Examples with VolumeManager org.apache.accumulo.server.fs.VolumeManager used on opensource projects

Example 11 with VolumeManager

use of org.apache.accumulo.server.fs.VolumeManager in project accumulo by apache.

the class DfsLogger method open.

/**
 * Opens a Write-Ahead Log file and writes the necessary header information and OPEN entry to the file. The file is ready to be used for ingest if this method
 * returns successfully. If an exception is thrown from this method, it is the callers responsibility to ensure that {@link #close()} is called to prevent
 * leaking the file handle and/or syncing thread.
 *
 * @param address
 *          The address of the host using this WAL
 */
public synchronized void open(String address) throws IOException {
    String filename = UUID.randomUUID().toString();
    log.debug("Address is {}", address);
    String logger = Joiner.on("+").join(address.split(":"));
    log.debug("DfsLogger.open() begin");
    VolumeManager fs = conf.getFileSystem();
    VolumeChooserEnvironment chooserEnv = new VolumeChooserEnvironment(ChooserScope.LOGGER);
    logPath = fs.choose(chooserEnv, ServerConstants.getBaseUris()) + Path.SEPARATOR + ServerConstants.WAL_DIR + Path.SEPARATOR + logger + Path.SEPARATOR + filename;
    metaReference = toString();
    LoggerOperation op = null;
    try {
        short replication = (short) conf.getConfiguration().getCount(Property.TSERV_WAL_REPLICATION);
        if (replication == 0)
            replication = fs.getDefaultReplication(new Path(logPath));
        long blockSize = getWalBlockSize(conf.getConfiguration());
        if (conf.getConfiguration().getBoolean(Property.TSERV_WAL_SYNC))
            logFile = fs.createSyncable(new Path(logPath), 0, replication, blockSize);
        else
            logFile = fs.create(new Path(logPath), true, 0, replication, blockSize);
        sync = logFile.getClass().getMethod("hsync");
        flush = logFile.getClass().getMethod("hflush");
        // Initialize the crypto operations.
        org.apache.accumulo.core.security.crypto.CryptoModule cryptoModule = org.apache.accumulo.core.security.crypto.CryptoModuleFactory.getCryptoModule(conf.getConfiguration().get(Property.CRYPTO_MODULE_CLASS));
        // Initialize the log file with a header and the crypto params used to set up this log file.
        logFile.write(LOG_FILE_HEADER_V3.getBytes(UTF_8));
        CryptoModuleParameters params = CryptoModuleFactory.createParamsObjectFromAccumuloConfiguration(conf.getConfiguration());
        // Immediately update to the correct cipher. Doing this here keeps the CryptoModule independent of the writers using it
        if (params.getAllOptions().get(Property.CRYPTO_WAL_CIPHER_SUITE.getKey()) != null && !params.getAllOptions().get(Property.CRYPTO_WAL_CIPHER_SUITE.getKey()).equals("")) {
            params.setCipherSuite(params.getAllOptions().get(Property.CRYPTO_WAL_CIPHER_SUITE.getKey()));
        }
        NoFlushOutputStream nfos = new NoFlushOutputStream(logFile);
        params.setPlaintextOutputStream(nfos);
        // In order to bootstrap the reading of this file later, we have to record the CryptoModule that was used to encipher it here,
        // so that that crypto module can re-read its own parameters.
        logFile.writeUTF(conf.getConfiguration().get(Property.CRYPTO_MODULE_CLASS));
        params = cryptoModule.getEncryptingOutputStream(params);
        OutputStream encipheringOutputStream = params.getEncryptedOutputStream();
        // another data OutputStream.
        if (encipheringOutputStream == nfos) {
            log.debug("No enciphering, using raw output stream");
            encryptingLogFile = nfos;
        } else {
            log.debug("Enciphering found, wrapping in DataOutputStream");
            encryptingLogFile = new DataOutputStream(encipheringOutputStream);
        }
        LogFileKey key = new LogFileKey();
        key.event = OPEN;
        key.tserverSession = filename;
        key.filename = filename;
        op = logFileData(Collections.singletonList(new Pair<>(key, EMPTY)), Durability.SYNC);
    } catch (Exception ex) {
        if (logFile != null)
            logFile.close();
        logFile = null;
        encryptingLogFile = null;
        throw new IOException(ex);
    }
    syncThread = new Daemon(new LoggingRunnable(log, new LogSyncingTask()));
    syncThread.setName("Accumulo WALog thread " + toString());
    syncThread.start();
    op.await();
    log.debug("Got new write-ahead log: {}", this);
}

Example 12 with VolumeManager

use of org.apache.accumulo.server.fs.VolumeManager in project accumulo by apache.

the class Initialize method execute.

@Override
public void execute(final String[] args) {
    Opts opts = new Opts();
    opts.parseArgs("accumulo init", args);
    try {
        zoo = ZooReaderWriter.getInstance();
        AccumuloConfiguration acuConf = SiteConfiguration.getInstance();
        SecurityUtil.serverLogin(acuConf);
        Configuration conf = CachedConfiguration.getInstance();
        VolumeManager fs = VolumeManagerImpl.get(acuConf);
        if (opts.resetSecurity) {
            log.info("Resetting security on accumulo.");
            Instance instance = HdfsZooInstance.getInstance();
            AccumuloServerContext context = new AccumuloServerContext(instance, new ServerConfigurationFactory(instance));
            if (isInitialized(fs)) {
                if (!opts.forceResetSecurity) {
                    ConsoleReader c = getConsoleReader();
                    String userEnteredName = c.readLine("WARNING: This will remove all users from Accumulo! If you wish to proceed enter the instance name: ");
                    if (userEnteredName != null && !instance.getInstanceName().equals(userEnteredName)) {
                        log.error("Aborted reset security: Instance name did not match current instance.");
                        return;
                    }
                }
                final String rootUser = getRootUserName(opts);
                opts.rootpass = getRootPassword(opts, rootUser);
                initSecurity(context, opts, HdfsZooInstance.getInstance().getInstanceID(), rootUser);
            } else {
                log.error("FATAL: Attempted to reset security on accumulo before it was initialized");
            }
        }
        if (opts.addVolumes) {
            addVolumes(fs);
        }
        if (!opts.resetSecurity && !opts.addVolumes)
            if (!doInit(opts, conf, fs))
                System.exit(-1);
    } catch (Exception e) {
        log.error("Fatal exception", e);
        throw new RuntimeException(e);
    }
}

Example 13 with VolumeManager

use of org.apache.accumulo.server.fs.VolumeManager in project accumulo by apache.

the class HdfsZooInstance method _getInstanceID.

private static synchronized void _getInstanceID() {
    if (instanceId == null) {
        AccumuloConfiguration acuConf = SiteConfiguration.getInstance();
        // InstanceID should be the same across all volumes, so just choose one
        VolumeManager fs;
        try {
            fs = VolumeManagerImpl.get();
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
        Path instanceIdPath = Accumulo.getAccumuloInstanceIdPath(fs);
        log.trace("Looking for instanceId from {}", instanceIdPath);
        String instanceIdFromFile = ZooUtil.getInstanceIDFromHdfs(instanceIdPath, acuConf);
        instanceId = instanceIdFromFile;
    }
}

Example 14 with VolumeManager

use of org.apache.accumulo.server.fs.VolumeManager in project accumulo by apache.

the class ZooConfigurationFactory method getInstance.

/**
 * Gets a configuration object for the given instance with the given parent. Repeated calls will return the same object.
 *
 * @param inst
 *          instance; if null, instance is determined from HDFS
 * @param zcf
 *          {@link ZooCacheFactory} for building {@link ZooCache} to contact ZooKeeper (required)
 * @param parent
 *          parent configuration (required)
 * @return configuration
 */
ZooConfiguration getInstance(Instance inst, ZooCacheFactory zcf, AccumuloConfiguration parent) {
    String instanceId;
    if (inst == null) {
        // InstanceID should be the same across all volumes, so just choose one
        VolumeManager fs;
        try {
            fs = VolumeManagerImpl.get();
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
        Path instanceIdPath = Accumulo.getAccumuloInstanceIdPath(fs);
        instanceId = ZooUtil.getInstanceIDFromHdfs(instanceIdPath, parent);
    } else {
        instanceId = inst.getInstanceID();
    }
    ZooConfiguration config;
    synchronized (instances) {
        config = instances.get(instanceId);
        if (config == null) {
            ZooCache propCache;
            // The purpose of this watcher is a hack. It forces the creation on a new zoocache instead of using a shared one. This was done so that the zoocache
            // would update less, causing the configuration update count to changes less.
            Watcher watcher = new Watcher() {

                @Override
                public void process(WatchedEvent arg0) {
                }
            };
            if (inst == null) {
                propCache = zcf.getZooCache(parent.get(Property.INSTANCE_ZK_HOST), (int) parent.getTimeInMillis(Property.INSTANCE_ZK_TIMEOUT), watcher);
            } else {
                propCache = zcf.getZooCache(inst.getZooKeepers(), inst.getZooKeepersSessionTimeOut(), watcher);
            }
            config = new ZooConfiguration(instanceId, propCache, parent);
            instances.put(instanceId, config);
        }
    }
    return config;
}

Example 15 with VolumeManager

use of org.apache.accumulo.server.fs.VolumeManager in project accumulo by apache.

the class ReplicationProcessorTest method peerTypeExtractionFromConfiguration.

@Test
public void peerTypeExtractionFromConfiguration() {
    Instance inst = EasyMock.createMock(Instance.class);
    VolumeManager fs = EasyMock.createMock(VolumeManager.class);
    Credentials creds = new Credentials("foo", new PasswordToken("bar"));
    ClientContext context = new ClientContext(inst, creds, ClientConfiguration.create());
    Map<String, String> data = new HashMap<>();
    String peerName = "peer";
    String configuration = "java.lang.String,foo";
    data.put(Property.REPLICATION_PEERS + peerName, configuration);
    ConfigurationCopy conf = new ConfigurationCopy(data);
    ReplicationProcessor proc = new ReplicationProcessor(context, conf, fs);
    Assert.assertEquals(configuration, proc.getPeerType(peerName));
}