Examples with AccumuloConfiguration org.apache.accumulo.core.conf.AccumuloConfiguration used on opensource projects

Example 86 with AccumuloConfiguration

use of org.apache.accumulo.core.conf.AccumuloConfiguration in project accumulo by apache.

the class TServerUtils method startServer.

/**
 * Start a server, at the given port, or higher, if that port is not available.
 *
 * @param context
 *          RPC configuration
 * @param portHintProperty
 *          the port to attempt to open, can be zero, meaning "any available port"
 * @param processor
 *          the service to be started
 * @param serverName
 *          the name of the class that is providing the service
 * @param threadName
 *          name this service's thread for better debugging
 * @param portSearchProperty
 *          A boolean Property to control if port-search should be used, or null to disable
 * @param minThreadProperty
 *          A Property to control the minimum number of threads in the pool
 * @param timeBetweenThreadChecksProperty
 *          A Property to control the amount of time between checks to resize the thread pool
 * @param maxMessageSizeProperty
 *          A Property to control the maximum Thrift message size accepted
 * @return the server object created, and the port actually used
 * @throws UnknownHostException
 *           when we don't know our own address
 */
public static ServerAddress startServer(ServerContext context, String hostname, Property portHintProperty, TProcessor processor, String serverName, String threadName, Property portSearchProperty, Property minThreadProperty, Property threadTimeOutProperty, Property timeBetweenThreadChecksProperty, Property maxMessageSizeProperty) throws UnknownHostException {
    final AccumuloConfiguration config = context.getConfiguration();
    final IntStream portHint = config.getPortStream(portHintProperty);
    int minThreads = 2;
    if (minThreadProperty != null) {
        minThreads = config.getCount(minThreadProperty);
    }
    long threadTimeOut = ThreadPools.DEFAULT_TIMEOUT_MILLISECS;
    if (threadTimeOutProperty != null) {
        threadTimeOut = config.getTimeInMillis(threadTimeOutProperty);
    }
    long timeBetweenThreadChecks = 1000;
    if (timeBetweenThreadChecksProperty != null) {
        timeBetweenThreadChecks = config.getTimeInMillis(timeBetweenThreadChecksProperty);
    }
    long maxMessageSize = 10_000_000;
    if (maxMessageSizeProperty != null) {
        maxMessageSize = config.getAsBytes(maxMessageSizeProperty);
    }
    boolean portSearch = false;
    if (portSearchProperty != null) {
        portSearch = config.getBoolean(portSearchProperty);
    }
    final ThriftServerType serverType = context.getThriftServerType();
    if (serverType == ThriftServerType.SASL) {
        processor = updateSaslProcessor(serverType, processor);
    }
    // create the TimedProcessor outside the port search loop so we don't try to
    // register the same
    // metrics mbean more than once
    TimedProcessor timedProcessor = new TimedProcessor(config, processor, serverName, threadName);
    HostAndPort[] addresses = getHostAndPorts(hostname, portHint);
    try {
        return TServerUtils.startTServer(serverType, timedProcessor, serverName, threadName, minThreads, threadTimeOut, config, timeBetweenThreadChecks, maxMessageSize, context.getServerSslParams(), context.getSaslParams(), context.getClientTimeoutInMillis(), addresses);
    } catch (TTransportException e) {
        if (portSearch) {
            // Build a list of reserved ports - as identified by properties of type PropertyType.PORT
            Map<Integer, Property> reservedPorts = getReservedPorts(config, portHintProperty);
            HostAndPort last = addresses[addresses.length - 1];
            // Search sequentially over the next 1000 ports
            for (int port = last.getPort() + 1; port < last.getPort() + 1001; port++) {
                if (reservedPorts.containsKey(port)) {
                    log.debug("During port search, skipping reserved port {} - property {} ({})", port, reservedPorts.get(port).getKey(), reservedPorts.get(port).getDescription());
                    continue;
                }
                if (PortRange.VALID_RANGE.isBefore(port)) {
                    break;
                }
                try {
                    HostAndPort addr = HostAndPort.fromParts(hostname, port);
                    return TServerUtils.startTServer(serverType, timedProcessor, serverName, threadName, minThreads, threadTimeOut, config, timeBetweenThreadChecks, maxMessageSize, context.getServerSslParams(), context.getSaslParams(), context.getClientTimeoutInMillis(), addr);
                } catch (TTransportException tte) {
                    log.info("Unable to use port {}, retrying. (Thread Name = {})", port, threadName);
                }
            }
            log.error("Unable to start TServer", e);
            throw new UnknownHostException("Unable to find a listen port");
        } else {
            log.error("Unable to start TServer", e);
            throw new UnknownHostException("Unable to find a listen port");
        }
    }
}

Example 87 with AccumuloConfiguration

use of org.apache.accumulo.core.conf.AccumuloConfiguration in project accumulo by apache.

the class InMemoryMap method delete.

public void delete(long waitTime) {
    synchronized (this) {
        if (deleted)
            throw new IllegalStateException("Double delete");
        deleted = true;
    }
    long t1 = System.currentTimeMillis();
    while (!activeIters.isEmpty() && System.currentTimeMillis() - t1 < waitTime) {
        sleepUninterruptibly(50, TimeUnit.MILLISECONDS);
    }
    if (!activeIters.isEmpty()) {
        // dump memmap exactly as is to a tmp file on disk, and switch scans to that temp file
        try {
            Configuration conf = context.getHadoopConf();
            FileSystem fs = FileSystem.getLocal(conf);
            String tmpFile = memDumpDir + "/memDump" + UUID.randomUUID() + "." + RFile.EXTENSION;
            Configuration newConf = new Configuration(conf);
            newConf.setInt("io.seqfile.compress.blocksize", 100000);
            AccumuloConfiguration aconf = context.getConfiguration();
            if (getOrCreateSampler() != null) {
                aconf = createSampleConfig(aconf);
            }
            FileSKVWriter out = new RFileOperations().newWriterBuilder().forFile(tmpFile, fs, newConf, context.getCryptoService()).withTableConfiguration(aconf).build();
            InterruptibleIterator iter = map.skvIterator(null);
            HashSet<ByteSequence> allfams = new HashSet<>();
            for (Entry<String, Set<ByteSequence>> entry : lggroups.entrySet()) {
                allfams.addAll(entry.getValue());
                out.startNewLocalityGroup(entry.getKey(), entry.getValue());
                iter.seek(new Range(), entry.getValue(), true);
                dumpLocalityGroup(out, iter);
            }
            out.startDefaultLocalityGroup();
            iter.seek(new Range(), allfams, false);
            dumpLocalityGroup(out, iter);
            out.close();
            log.debug("Created mem dump file {}", tmpFile);
            memDumpFile = tmpFile;
            synchronized (activeIters) {
                for (MemoryIterator mi : activeIters) {
                    mi.switchNow();
                }
            }
            // rely on unix behavior that file will be deleted when last
            // reader closes it
            fs.delete(new Path(memDumpFile), true);
        } catch (IOException ioe) {
            log.error("Failed to create mem dump file", ioe);
            while (!activeIters.isEmpty()) {
                sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
            }
        }
    }
    SimpleMap tmpMap = map;
    synchronized (this) {
        map = null;
    }
    tmpMap.delete();
}

Example 88 with AccumuloConfiguration

use of org.apache.accumulo.core.conf.AccumuloConfiguration in project accumulo by apache.

the class InMemoryMap method createSampleConfig.

private AccumuloConfiguration createSampleConfig(AccumuloConfiguration siteConf) {
    ConfigurationCopy confCopy = new ConfigurationCopy(Iterables.filter(siteConf, input -> !input.getKey().startsWith(Property.TABLE_SAMPLER.getKey())));
    for (Entry<String, String> entry : samplerRef.get().getFirst().toTablePropertiesMap().entrySet()) {
        confCopy.set(entry.getKey(), entry.getValue());
    }
    siteConf = confCopy;
    return siteConf;
}

Example 89 with AccumuloConfiguration

use of org.apache.accumulo.core.conf.AccumuloConfiguration in project accumulo by apache.

the class WebViews method addExternalResources.

/**
 * Get HTML for external CSS and JS resources from configuration. See ACCUMULO-4739
 *
 * @param model
 *          map of the MVC model
 */
private void addExternalResources(Map<String, Object> model) {
    AccumuloConfiguration conf = monitor.getContext().getConfiguration();
    String resourcesProperty = conf.get(Property.MONITOR_RESOURCES_EXTERNAL);
    if (resourcesProperty.isBlank()) {
        return;
    }
    List<String> monitorResources = new ArrayList<>();
    ObjectMapper objectMapper = new ObjectMapper();
    try {
        Collections.addAll(monitorResources, objectMapper.readValue(resourcesProperty, String[].class));
    } catch (IOException e) {
        log.error("Error Monitor Resources config property {}: {}", Property.MONITOR_RESOURCES_EXTERNAL, e);
        return;
    }
    if (!monitorResources.isEmpty()) {
        model.put("externalResources", monitorResources);
    }
}

Example 90 with AccumuloConfiguration

use of org.apache.accumulo.core.conf.AccumuloConfiguration in project accumulo by apache.

the class AccumuloReplicaSystem method replicate.

@SuppressFBWarnings(value = "PATH_TRAVERSAL_IN", justification = "path provided by admin")
@Override
public Status replicate(final Path p, final Status status, final ReplicationTarget target, final ReplicaSystemHelper helper) {
    final AccumuloConfiguration localConf = conf;
    log.debug("Replication RPC timeout is {}", localConf.get(Property.REPLICATION_RPC_TIMEOUT.getKey()));
    final String principal = getPrincipal(localConf, target);
    final File keytab;
    final String password;
    if (localConf.getBoolean(Property.INSTANCE_RPC_SASL_ENABLED)) {
        String keytabPath = getKeytab(localConf, target);
        keytab = new File(keytabPath);
        if (!keytab.exists() || !keytab.isFile()) {
            log.error("{} is not a regular file. Cannot login to replicate", keytabPath);
            return status;
        }
        password = null;
    } else {
        keytab = null;
        password = getPassword(localConf, target);
    }
    if (keytab != null) {
        try {
            final UserGroupInformation accumuloUgi = UserGroupInformation.getCurrentUser();
            // Get a UGI with the principal + keytab
            UserGroupInformation ugi = UserGroupInformation.loginUserFromKeytabAndReturnUGI(principal, keytab.getAbsolutePath());
            // Run inside a doAs to avoid nuking the Tserver's user
            return ugi.doAs((PrivilegedAction<Status>) () -> {
                KerberosToken token;
                try {
                    // Do *not* replace the current user
                    token = new KerberosToken(principal, keytab);
                } catch (IOException e) {
                    log.error("Failed to create KerberosToken", e);
                    return status;
                }
                ClientContext peerContext = getContextForPeer(localConf, target, principal, token);
                return _replicate(p, status, target, helper, localConf, peerContext, accumuloUgi);
            });
        } catch (IOException e) {
            // Can't log in, can't replicate
            log.error("Failed to perform local login", e);
            return status;
        }
    } else {
        // Simple case: make a password token, context and then replicate
        PasswordToken token = new PasswordToken(password);
        ClientContext peerContext = getContextForPeer(localConf, target, principal, token);
        return _replicate(p, status, target, helper, localConf, peerContext, null);
    }
}