Examples with Daemon org.apache.accumulo.core.util.Daemon used on opensource projects

Example 1 with Daemon

use of org.apache.accumulo.core.util.Daemon in project accumulo by apache.

the class TabletServer method config.

public void config(String hostname) {
    log.info("Tablet server starting on {}", hostname);
    majorCompactorThread = new Daemon(new LoggingRunnable(log, new MajorCompactor(getConfiguration())));
    majorCompactorThread.setName("Split/MajC initiator");
    majorCompactorThread.start();
    clientAddress = HostAndPort.fromParts(hostname, 0);
    try {
        AccumuloVFSClassLoader.getContextManager().setContextConfig(new ContextManager.DefaultContextsConfig() {

            @Override
            public Map<String, String> getVfsContextClasspathProperties() {
                return getConfiguration().getAllPropertiesWithPrefix(Property.VFS_CONTEXT_CLASSPATH_PROPERTY);
            }
        });
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
    // A task that cleans up unused classloader contexts
    Runnable contextCleaner = new Runnable() {

        @Override
        public void run() {
            Set<String> contextProperties = getServerConfigurationFactory().getSystemConfiguration().getAllPropertiesWithPrefix(Property.VFS_CONTEXT_CLASSPATH_PROPERTY).keySet();
            Set<String> configuredContexts = new HashSet<>();
            for (String prop : contextProperties) {
                configuredContexts.add(prop.substring(Property.VFS_CONTEXT_CLASSPATH_PROPERTY.name().length()));
            }
            try {
                AccumuloVFSClassLoader.getContextManager().removeUnusedContexts(configuredContexts);
            } catch (IOException e) {
                log.warn("{}", e.getMessage(), e);
            }
        }
    };
    AccumuloConfiguration aconf = getConfiguration();
    SimpleTimer.getInstance(aconf).schedule(contextCleaner, 60000, 60000);
    FileSystemMonitor.start(aconf, Property.TSERV_MONITOR_FS);
    Runnable gcDebugTask = new Runnable() {

        @Override
        public void run() {
            gcLogger.logGCInfo(getConfiguration());
        }
    };
    SimpleTimer.getInstance(aconf).schedule(gcDebugTask, 0, TIME_BETWEEN_GC_CHECKS);
    Runnable constraintTask = new Runnable() {

        @Override
        public void run() {
            ArrayList<Tablet> tablets;
            synchronized (onlineTablets) {
                tablets = new ArrayList<>(onlineTablets.values());
            }
            for (Tablet tablet : tablets) {
                tablet.checkConstraints();
            }
        }
    };
    SimpleTimer.getInstance(aconf).schedule(constraintTask, 0, 1000);
}

Example 2 with Daemon

use of org.apache.accumulo.core.util.Daemon in project accumulo by apache.

the class DfsLogger method open.

/**
 * Opens a Write-Ahead Log file and writes the necessary header information and OPEN entry to the file. The file is ready to be used for ingest if this method
 * returns successfully. If an exception is thrown from this method, it is the callers responsibility to ensure that {@link #close()} is called to prevent
 * leaking the file handle and/or syncing thread.
 *
 * @param address
 *          The address of the host using this WAL
 */
public synchronized void open(String address) throws IOException {
    String filename = UUID.randomUUID().toString();
    log.debug("Address is {}", address);
    String logger = Joiner.on("+").join(address.split(":"));
    log.debug("DfsLogger.open() begin");
    VolumeManager fs = conf.getFileSystem();
    VolumeChooserEnvironment chooserEnv = new VolumeChooserEnvironment(ChooserScope.LOGGER);
    logPath = fs.choose(chooserEnv, ServerConstants.getBaseUris()) + Path.SEPARATOR + ServerConstants.WAL_DIR + Path.SEPARATOR + logger + Path.SEPARATOR + filename;
    metaReference = toString();
    LoggerOperation op = null;
    try {
        short replication = (short) conf.getConfiguration().getCount(Property.TSERV_WAL_REPLICATION);
        if (replication == 0)
            replication = fs.getDefaultReplication(new Path(logPath));
        long blockSize = getWalBlockSize(conf.getConfiguration());
        if (conf.getConfiguration().getBoolean(Property.TSERV_WAL_SYNC))
            logFile = fs.createSyncable(new Path(logPath), 0, replication, blockSize);
        else
            logFile = fs.create(new Path(logPath), true, 0, replication, blockSize);
        sync = logFile.getClass().getMethod("hsync");
        flush = logFile.getClass().getMethod("hflush");
        // Initialize the crypto operations.
        org.apache.accumulo.core.security.crypto.CryptoModule cryptoModule = org.apache.accumulo.core.security.crypto.CryptoModuleFactory.getCryptoModule(conf.getConfiguration().get(Property.CRYPTO_MODULE_CLASS));
        // Initialize the log file with a header and the crypto params used to set up this log file.
        logFile.write(LOG_FILE_HEADER_V3.getBytes(UTF_8));
        CryptoModuleParameters params = CryptoModuleFactory.createParamsObjectFromAccumuloConfiguration(conf.getConfiguration());
        // Immediately update to the correct cipher. Doing this here keeps the CryptoModule independent of the writers using it
        if (params.getAllOptions().get(Property.CRYPTO_WAL_CIPHER_SUITE.getKey()) != null && !params.getAllOptions().get(Property.CRYPTO_WAL_CIPHER_SUITE.getKey()).equals("")) {
            params.setCipherSuite(params.getAllOptions().get(Property.CRYPTO_WAL_CIPHER_SUITE.getKey()));
        }
        NoFlushOutputStream nfos = new NoFlushOutputStream(logFile);
        params.setPlaintextOutputStream(nfos);
        // In order to bootstrap the reading of this file later, we have to record the CryptoModule that was used to encipher it here,
        // so that that crypto module can re-read its own parameters.
        logFile.writeUTF(conf.getConfiguration().get(Property.CRYPTO_MODULE_CLASS));
        params = cryptoModule.getEncryptingOutputStream(params);
        OutputStream encipheringOutputStream = params.getEncryptedOutputStream();
        // another data OutputStream.
        if (encipheringOutputStream == nfos) {
            log.debug("No enciphering, using raw output stream");
            encryptingLogFile = nfos;
        } else {
            log.debug("Enciphering found, wrapping in DataOutputStream");
            encryptingLogFile = new DataOutputStream(encipheringOutputStream);
        }
        LogFileKey key = new LogFileKey();
        key.event = OPEN;
        key.tserverSession = filename;
        key.filename = filename;
        op = logFileData(Collections.singletonList(new Pair<>(key, EMPTY)), Durability.SYNC);
    } catch (Exception ex) {
        if (logFile != null)
            logFile.close();
        logFile = null;
        encryptingLogFile = null;
        throw new IOException(ex);
    }
    syncThread = new Daemon(new LoggingRunnable(log, new LogSyncingTask()));
    syncThread.setName("Accumulo WALog thread " + toString());
    syncThread.start();
    op.await();
    log.debug("Got new write-ahead log: {}", this);
}

Example 3 with Daemon

use of org.apache.accumulo.core.util.Daemon in project accumulo by apache.

the class LogService method startLogListener.

/**
 * Place the host:port advertisement for the Monitor's Log4j listener in ZooKeeper
 *
 * @param conf
 *          configuration for the instance
 * @param instanceId
 *          instanceId for the instance
 * @param hostAddress
 *          Address that monitor process is bound to
 */
public static void startLogListener(AccumuloConfiguration conf, String instanceId, String hostAddress) {
    try {
        SocketServer server = new SocketServer(conf.getPort(Property.MONITOR_LOG4J_PORT)[0]);
        // getLocalPort will return the actual ephemeral port used when '0' was provided.
        String logForwardingAddr = hostAddress + ":" + server.getLocalPort();
        log.debug("Setting monitor log4j log-forwarding address to: {}", logForwardingAddr);
        final String path = ZooUtil.getRoot(instanceId) + Constants.ZMONITOR_LOG4J_ADDR;
        final ZooReaderWriter zoo = ZooReaderWriter.getInstance();
        // Delete before we try to re-create in case the previous session hasn't yet expired
        try {
            zoo.delete(path, -1);
        } catch (KeeperException e) {
            // We don't care if the node is already gone
            if (!KeeperException.Code.NONODE.equals(e.code())) {
                throw e;
            }
        }
        zoo.putEphemeralData(path, logForwardingAddr.getBytes(UTF_8));
        new Daemon(server).start();
    } catch (Throwable t) {
        log.info("Unable to start/advertise Log4j listener for log-forwarding to monitor", t);
    }
}

Example 4 with Daemon

use of org.apache.accumulo.core.util.Daemon in project accumulo by apache.

the class ThriftTransportPool method getInstance.

public static ThriftTransportPool getInstance() {
    if (daemonStarted.compareAndSet(false, true)) {
        CountDownLatch closerExitLatch = new CountDownLatch(1);
        new Daemon(new Closer(instance, closerExitLatch), "Thrift Connection Pool Checker").start();
        instance.setCloserExitLatch(closerExitLatch);
    }
    return instance;
}

Example 5 with Daemon

use of org.apache.accumulo.core.util.Daemon in project accumulo by apache.

the class TServerUtils method startTServer.

/**
 * Start the appropriate Thrift server (SSL or non-blocking server) for the given parameters. Non-null SSL parameters will cause an SSL server to be started.
 *
 * @return A ServerAddress encapsulating the Thrift server created and the host/port which it is bound to.
 */
public static ServerAddress startTServer(ThriftServerType serverType, TimedProcessor processor, TProtocolFactory protocolFactory, String serverName, String threadName, int numThreads, int numSTThreads, long timeBetweenThreadChecks, long maxMessageSize, SslConnectionParams sslParams, SaslServerConnectionParams saslParams, long serverSocketTimeout, HostAndPort... addresses) throws TTransportException {
    // This is presently not supported. It's hypothetically possible, I believe, to work, but it would require changes in how the transports
    // work at the Thrift layer to ensure that both the SSL and SASL handshakes function. SASL's quality of protection addresses privacy issues.
    checkArgument(!(sslParams != null && saslParams != null), "Cannot start a Thrift server using both SSL and SASL");
    ServerAddress serverAddress = null;
    for (HostAndPort address : addresses) {
        try {
            switch(serverType) {
                case SSL:
                    log.debug("Instantiating SSL Thrift server");
                    serverAddress = createSslThreadPoolServer(address, processor, protocolFactory, serverSocketTimeout, sslParams, serverName, numThreads, numSTThreads, timeBetweenThreadChecks);
                    break;
                case SASL:
                    log.debug("Instantiating SASL Thrift server");
                    serverAddress = createSaslThreadPoolServer(address, processor, protocolFactory, serverSocketTimeout, saslParams, serverName, threadName, numThreads, numSTThreads, timeBetweenThreadChecks);
                    break;
                case THREADPOOL:
                    log.debug("Instantiating unsecure TThreadPool Thrift server");
                    serverAddress = createBlockingServer(address, processor, protocolFactory, maxMessageSize, serverName, numThreads, numSTThreads, timeBetweenThreadChecks);
                    break;
                // Intentional passthrough -- Our custom wrapper around HsHa is the default
                case CUSTOM_HS_HA:
                default:
                    log.debug("Instantiating default, unsecure custom half-async Thrift server");
                    serverAddress = createNonBlockingServer(address, processor, protocolFactory, serverName, threadName, numThreads, numSTThreads, timeBetweenThreadChecks, maxMessageSize);
            }
            break;
        } catch (TTransportException e) {
            log.warn("Error attempting to create server at {}. Error: {}", address.toString(), e.getMessage());
        }
    }
    if (null == serverAddress) {
        throw new TTransportException("Unable to create server on addresses: " + Arrays.toString(addresses));
    }
    final TServer finalServer = serverAddress.server;
    Runnable serveTask = new Runnable() {

        @Override
        public void run() {
            try {
                finalServer.serve();
            } catch (Error e) {
                Halt.halt("Unexpected error in TThreadPoolServer " + e + ", halting.", 1);
            }
        }
    };
    serveTask = new LoggingRunnable(TServerUtils.log, serveTask);
    Thread thread = new Daemon(serveTask, threadName);
    thread.start();
    // check for the special "bind to everything address"
    if (serverAddress.address.getHost().equals("0.0.0.0")) {
        // can't get the address from the bind, so we'll do our best to invent our hostname
        try {
            serverAddress = new ServerAddress(finalServer, HostAndPort.fromParts(InetAddress.getLocalHost().getHostName(), serverAddress.address.getPort()));
        } catch (UnknownHostException e) {
            throw new TTransportException(e);
        }
    }
    return serverAddress;
}