Examples with Daemon org.apache.hadoop.util.Daemon used on opensource projects

Example 16 with Daemon

use of org.apache.hadoop.util.Daemon in project hadoop by apache.

the class TestDelegationToken method testParallelDelegationTokenCreation.

@Test
public void testParallelDelegationTokenCreation() throws Exception {
    final TestDelegationTokenSecretManager dtSecretManager = new TestDelegationTokenSecretManager(2000, 24 * 60 * 60 * 1000, 7 * 24 * 60 * 60 * 1000, 2000);
    try {
        dtSecretManager.startThreads();
        int numThreads = 100;
        final int numTokensPerThread = 100;
        class tokenIssuerThread implements Runnable {

            @Override
            public void run() {
                for (int i = 0; i < numTokensPerThread; i++) {
                    generateDelegationToken(dtSecretManager, "auser", "arenewer");
                    try {
                        Thread.sleep(250);
                    } catch (Exception e) {
                    }
                }
            }
        }
        Thread[] issuers = new Thread[numThreads];
        for (int i = 0; i < numThreads; i++) {
            issuers[i] = new Daemon(new tokenIssuerThread());
            issuers[i].start();
        }
        for (int i = 0; i < numThreads; i++) {
            issuers[i].join();
        }
        Map<TestDelegationTokenIdentifier, DelegationTokenInformation> tokenCache = dtSecretManager.getAllTokens();
        Assert.assertEquals(numTokensPerThread * numThreads, tokenCache.size());
        Iterator<TestDelegationTokenIdentifier> iter = tokenCache.keySet().iterator();
        while (iter.hasNext()) {
            TestDelegationTokenIdentifier id = iter.next();
            DelegationTokenInformation info = tokenCache.get(id);
            Assert.assertTrue(info != null);
            DelegationKey key = dtSecretManager.getKey(id);
            Assert.assertTrue(key != null);
            byte[] storedPassword = dtSecretManager.retrievePassword(id);
            byte[] password = dtSecretManager.createPassword(id, key);
            Assert.assertTrue(Arrays.equals(password, storedPassword));
            //verify by secret manager api
            dtSecretManager.verifyToken(id, password);
        }
    } finally {
        dtSecretManager.stopThreads();
    }
}

Example 17 with Daemon

use of org.apache.hadoop.util.Daemon in project hadoop by apache.

the class PeerCache method startExpiryDaemon.

private synchronized void startExpiryDaemon() {
    // start daemon only if not already started
    if (isDaemonStarted()) {
        return;
    }
    daemon = new Daemon(new Runnable() {

        @Override
        public void run() {
            try {
                PeerCache.this.run();
            } catch (InterruptedException e) {
            //noop
            } finally {
                PeerCache.this.clear();
            }
        }

        @Override
        public String toString() {
            return String.valueOf(PeerCache.this);
        }
    });
    daemon.start();
}

Example 18 with Daemon

use of org.apache.hadoop.util.Daemon in project hadoop by apache.

the class DataXceiverServer method run.

@Override
public void run() {
    Peer peer = null;
    while (datanode.shouldRun && !datanode.shutdownForUpgrade) {
        try {
            peer = peerServer.accept();
            // Make sure the xceiver count is not exceeded
            int curXceiverCount = datanode.getXceiverCount();
            if (curXceiverCount > maxXceiverCount) {
                throw new IOException("Xceiver count " + curXceiverCount + " exceeds the limit of concurrent xcievers: " + maxXceiverCount);
            }
            new Daemon(datanode.threadGroup, DataXceiver.create(peer, datanode, this)).start();
        } catch (SocketTimeoutException ignored) {
        // wake up to see if should continue to run
        } catch (AsynchronousCloseException ace) {
            // but not in other circumstances
            if (datanode.shouldRun && !datanode.shutdownForUpgrade) {
                LOG.warn(datanode.getDisplayName() + ":DataXceiverServer: ", ace);
            }
        } catch (IOException ie) {
            IOUtils.cleanup(null, peer);
            LOG.warn(datanode.getDisplayName() + ":DataXceiverServer: ", ie);
        } catch (OutOfMemoryError ie) {
            IOUtils.cleanup(null, peer);
            // DataNode can run out of memory if there is too many transfers.
            // Log the event, Sleep for 30 seconds, other transfers may complete by
            // then.
            LOG.error("DataNode is out of memory. Will retry in 30 seconds.", ie);
            try {
                Thread.sleep(30 * 1000);
            } catch (InterruptedException e) {
            // ignore
            }
        } catch (Throwable te) {
            LOG.error(datanode.getDisplayName() + ":DataXceiverServer: Exiting due to: ", te);
            datanode.shouldRun = false;
        }
    }
    // Close the server to stop reception of more requests.
    try {
        peerServer.close();
        closed = true;
    } catch (IOException ie) {
        LOG.warn(datanode.getDisplayName() + " :DataXceiverServer: close exception", ie);
    }
    // if in restart prep stage, notify peers before closing them.
    if (datanode.shutdownForUpgrade) {
        restartNotifyPeers();
        // Each thread needs some time to process it. If a thread needs
        // to send an OOB message to the client, but blocked on network for
        // long time, we need to force its termination.
        LOG.info("Shutting down DataXceiverServer before restart");
        // Allow roughly up to 2 seconds.
        for (int i = 0; getNumPeers() > 0 && i < 10; i++) {
            try {
                Thread.sleep(200);
            } catch (InterruptedException e) {
            // ignore
            }
        }
    }
    // Close all peers.
    closeAllPeers();
}

Example 19 with Daemon

use of org.apache.hadoop.util.Daemon in project hadoop by apache.

the class DataNode method initDataXceiver.

private void initDataXceiver() throws IOException {
    // find free port or use privileged port provided
    TcpPeerServer tcpPeerServer;
    if (secureResources != null) {
        tcpPeerServer = new TcpPeerServer(secureResources);
    } else {
        int backlogLength = getConf().getInt(CommonConfigurationKeysPublic.IPC_SERVER_LISTEN_QUEUE_SIZE_KEY, CommonConfigurationKeysPublic.IPC_SERVER_LISTEN_QUEUE_SIZE_DEFAULT);
        tcpPeerServer = new TcpPeerServer(dnConf.socketWriteTimeout, DataNode.getStreamingAddr(getConf()), backlogLength);
    }
    if (dnConf.getTransferSocketRecvBufferSize() > 0) {
        tcpPeerServer.setReceiveBufferSize(dnConf.getTransferSocketRecvBufferSize());
    }
    streamingAddr = tcpPeerServer.getStreamingAddr();
    LOG.info("Opened streaming server at " + streamingAddr);
    this.threadGroup = new ThreadGroup("dataXceiverServer");
    xserver = new DataXceiverServer(tcpPeerServer, getConf(), this);
    this.dataXceiverServer = new Daemon(threadGroup, xserver);
    // auto destroy when empty
    this.threadGroup.setDaemon(true);
    if (getConf().getBoolean(HdfsClientConfigKeys.Read.ShortCircuit.KEY, HdfsClientConfigKeys.Read.ShortCircuit.DEFAULT) || getConf().getBoolean(HdfsClientConfigKeys.DFS_CLIENT_DOMAIN_SOCKET_DATA_TRAFFIC, HdfsClientConfigKeys.DFS_CLIENT_DOMAIN_SOCKET_DATA_TRAFFIC_DEFAULT)) {
        DomainPeerServer domainPeerServer = getDomainPeerServer(getConf(), streamingAddr.getPort());
        if (domainPeerServer != null) {
            this.localDataXceiverServer = new Daemon(threadGroup, new DataXceiverServer(domainPeerServer, getConf(), this));
            LOG.info("Listening on UNIX domain socket: " + domainPeerServer.getBindPath());
        }
    }
    this.shortCircuitRegistry = new ShortCircuitRegistry(getConf());
}

Example 20 with Daemon

use of org.apache.hadoop.util.Daemon in project hadoop by apache.

the class BlockPoolSliceStorage method doFinalize.

/*
   * Finalize the block pool storage by deleting <BP>/previous directory
   * that holds the snapshot.
   */
void doFinalize(File dnCurDir) throws IOException {
    File bpRoot = getBpRoot(blockpoolID, dnCurDir);
    StorageDirectory bpSd = new StorageDirectory(bpRoot);
    // block pool level previous directory
    File prevDir = bpSd.getPreviousDir();
    if (!prevDir.exists()) {
        // already finalized
        return;
    }
    final String dataDirPath = bpSd.getRoot().getCanonicalPath();
    LOG.info("Finalizing upgrade for storage directory " + dataDirPath + ".\n   cur LV = " + this.getLayoutVersion() + "; cur CTime = " + this.getCTime());
    assert bpSd.getCurrentDir().exists() : "Current directory must exist.";
    // rename previous to finalized.tmp
    final File tmpDir = bpSd.getFinalizedTmp();
    rename(prevDir, tmpDir);
    // delete finalized.tmp dir in a separate thread
    new Daemon(new Runnable() {

        @Override
        public void run() {
            try {
                deleteDir(tmpDir);
            } catch (IOException ex) {
                LOG.error("Finalize upgrade for " + dataDirPath + " failed.", ex);
            }
            LOG.info("Finalize upgrade for " + dataDirPath + " is complete.");
        }

        @Override
        public String toString() {
            return "Finalize " + dataDirPath;
        }
    }).start();
}