Examples with ClientContext org.apache.accumulo.core.client.impl.ClientContext used on opensource projects

Example 21 with ClientContext

use of org.apache.accumulo.core.client.impl.ClientContext in project accumulo by apache.

the class AbstractInputFormat method getSplits.

/**
 * Gets the splits of the tables that have been set on the job by reading the metadata table for the specified ranges.
 *
 * @return the splits from the tables based on the ranges.
 * @throws java.io.IOException
 *           if a table set on the job doesn't exist or an error occurs initializing the tablet locator
 */
@Override
public InputSplit[] getSplits(JobConf job, int numSplits) throws IOException {
    Level logLevel = getLogLevel(job);
    log.setLevel(logLevel);
    validateOptions(job);
    Random random = new Random();
    LinkedList<InputSplit> splits = new LinkedList<>();
    Map<String, InputTableConfig> tableConfigs = getInputTableConfigs(job);
    for (Map.Entry<String, InputTableConfig> tableConfigEntry : tableConfigs.entrySet()) {
        String tableName = tableConfigEntry.getKey();
        InputTableConfig tableConfig = tableConfigEntry.getValue();
        Instance instance = getInstance(job);
        Table.ID tableId;
        // resolve table name to id once, and use id from this point forward
        if (DeprecationUtil.isMockInstance(instance)) {
            tableId = Table.ID.of("");
        } else {
            try {
                tableId = Tables.getTableId(instance, tableName);
            } catch (TableNotFoundException e) {
                throw new IOException(e);
            }
        }
        Authorizations auths = getScanAuthorizations(job);
        String principal = getPrincipal(job);
        AuthenticationToken token = getAuthenticationToken(job);
        boolean batchScan = InputConfigurator.isBatchScan(CLASS, job);
        boolean supportBatchScan = !(tableConfig.isOfflineScan() || tableConfig.shouldUseIsolatedScanners() || tableConfig.shouldUseLocalIterators());
        if (batchScan && !supportBatchScan)
            throw new IllegalArgumentException("BatchScanner optimization not available for offline scan, isolated, or local iterators");
        boolean autoAdjust = tableConfig.shouldAutoAdjustRanges();
        if (batchScan && !autoAdjust)
            throw new IllegalArgumentException("AutoAdjustRanges must be enabled when using BatchScanner optimization");
        List<Range> ranges = autoAdjust ? Range.mergeOverlapping(tableConfig.getRanges()) : tableConfig.getRanges();
        if (ranges.isEmpty()) {
            ranges = new ArrayList<>(1);
            ranges.add(new Range());
        }
        // get the metadata information for these ranges
        Map<String, Map<KeyExtent, List<Range>>> binnedRanges = new HashMap<>();
        TabletLocator tl;
        try {
            if (tableConfig.isOfflineScan()) {
                binnedRanges = binOfflineTable(job, tableId, ranges);
                while (binnedRanges == null) {
                    // Some tablets were still online, try again
                    // sleep randomly between 100 and 200 ms
                    sleepUninterruptibly(100 + random.nextInt(100), TimeUnit.MILLISECONDS);
                    binnedRanges = binOfflineTable(job, tableId, ranges);
                }
            } else {
                tl = InputConfigurator.getTabletLocator(CLASS, job, tableId);
                // its possible that the cache could contain complete, but old information about a tables tablets... so clear it
                tl.invalidateCache();
                ClientContext context = new ClientContext(getInstance(job), new Credentials(getPrincipal(job), getAuthenticationToken(job)), getClientConfiguration(job));
                while (!tl.binRanges(context, ranges, binnedRanges).isEmpty()) {
                    if (!DeprecationUtil.isMockInstance(instance)) {
                        String tableIdStr = tableId.canonicalID();
                        if (!Tables.exists(instance, tableId))
                            throw new TableDeletedException(tableIdStr);
                        if (Tables.getTableState(instance, tableId) == TableState.OFFLINE)
                            throw new TableOfflineException(instance, tableIdStr);
                    }
                    binnedRanges.clear();
                    log.warn("Unable to locate bins for specified ranges. Retrying.");
                    // sleep randomly between 100 and 200 ms
                    sleepUninterruptibly(100 + random.nextInt(100), TimeUnit.MILLISECONDS);
                    tl.invalidateCache();
                }
            }
        } catch (Exception e) {
            throw new IOException(e);
        }
        HashMap<Range, ArrayList<String>> splitsToAdd = null;
        if (!autoAdjust)
            splitsToAdd = new HashMap<>();
        HashMap<String, String> hostNameCache = new HashMap<>();
        for (Map.Entry<String, Map<KeyExtent, List<Range>>> tserverBin : binnedRanges.entrySet()) {
            String ip = tserverBin.getKey().split(":", 2)[0];
            String location = hostNameCache.get(ip);
            if (location == null) {
                InetAddress inetAddress = InetAddress.getByName(ip);
                location = inetAddress.getCanonicalHostName();
                hostNameCache.put(ip, location);
            }
            for (Map.Entry<KeyExtent, List<Range>> extentRanges : tserverBin.getValue().entrySet()) {
                Range ke = extentRanges.getKey().toDataRange();
                if (batchScan) {
                    // group ranges by tablet to be read by a BatchScanner
                    ArrayList<Range> clippedRanges = new ArrayList<>();
                    for (Range r : extentRanges.getValue()) clippedRanges.add(ke.clip(r));
                    BatchInputSplit split = new BatchInputSplit(tableName, tableId, clippedRanges, new String[] { location });
                    SplitUtils.updateSplit(split, instance, tableConfig, principal, token, auths, logLevel);
                    splits.add(split);
                } else {
                    // not grouping by tablet
                    for (Range r : extentRanges.getValue()) {
                        if (autoAdjust) {
                            // divide ranges into smaller ranges, based on the tablets
                            RangeInputSplit split = new RangeInputSplit(tableName, tableId.canonicalID(), ke.clip(r), new String[] { location });
                            SplitUtils.updateSplit(split, instance, tableConfig, principal, token, auths, logLevel);
                            split.setOffline(tableConfig.isOfflineScan());
                            split.setIsolatedScan(tableConfig.shouldUseIsolatedScanners());
                            split.setUsesLocalIterators(tableConfig.shouldUseLocalIterators());
                            splits.add(split);
                        } else {
                            // don't divide ranges
                            ArrayList<String> locations = splitsToAdd.get(r);
                            if (locations == null)
                                locations = new ArrayList<>(1);
                            locations.add(location);
                            splitsToAdd.put(r, locations);
                        }
                    }
                }
            }
        }
        if (!autoAdjust)
            for (Map.Entry<Range, ArrayList<String>> entry : splitsToAdd.entrySet()) {
                RangeInputSplit split = new RangeInputSplit(tableName, tableId.canonicalID(), entry.getKey(), entry.getValue().toArray(new String[0]));
                SplitUtils.updateSplit(split, instance, tableConfig, principal, token, auths, logLevel);
                split.setOffline(tableConfig.isOfflineScan());
                split.setIsolatedScan(tableConfig.shouldUseIsolatedScanners());
                split.setUsesLocalIterators(tableConfig.shouldUseLocalIterators());
                splits.add(split);
            }
    }
    return splits.toArray(new InputSplit[splits.size()]);
}

Example 22 with ClientContext

use of org.apache.accumulo.core.client.impl.ClientContext in project accumulo by apache.

the class MiniAccumuloClusterImpl method getMasterMonitorInfo.

/**
 * Get programmatic interface to information available in a normal monitor. XXX the returned structure won't contain information about the metadata table
 * until there is data in it. e.g. if you want to see the metadata table you should create a table.
 *
 * @since 1.6.1
 */
public MasterMonitorInfo getMasterMonitorInfo() throws AccumuloException, AccumuloSecurityException {
    MasterClientService.Iface client = null;
    while (true) {
        try {
            Instance instance = new ZooKeeperInstance(getClientConfig());
            ClientContext context = new ClientContext(instance, new Credentials("root", new PasswordToken("unchecked")), getClientConfig());
            client = MasterClient.getConnectionWithRetry(context);
            return client.getMasterStats(Tracer.traceInfo(), context.rpcCreds());
        } catch (ThriftSecurityException exception) {
            throw new AccumuloSecurityException(exception);
        } catch (ThriftNotActiveServiceException e) {
            // Let it loop, fetching a new location
            log.debug("Contacted a Master which is no longer active, retrying");
            sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
        } catch (TException exception) {
            throw new AccumuloException(exception);
        } finally {
            if (client != null) {
                MasterClient.close(client);
            }
        }
    }
}

Example 23 with ClientContext

use of org.apache.accumulo.core.client.impl.ClientContext in project accumulo by apache.

the class Admin method execute.

@Override
public void execute(final String[] args) {
    boolean everything;
    AdminOpts opts = new AdminOpts();
    JCommander cl = new JCommander(opts);
    cl.setProgramName("accumulo admin");
    CheckTabletsCommand checkTabletsCommand = new CheckTabletsCommand();
    cl.addCommand("checkTablets", checkTabletsCommand);
    ListInstancesCommand listIntancesOpts = new ListInstancesCommand();
    cl.addCommand("listInstances", listIntancesOpts);
    PingCommand pingCommand = new PingCommand();
    cl.addCommand("ping", pingCommand);
    DumpConfigCommand dumpConfigCommand = new DumpConfigCommand();
    cl.addCommand("dumpConfig", dumpConfigCommand);
    VolumesCommand volumesCommand = new VolumesCommand();
    cl.addCommand("volumes", volumesCommand);
    StopCommand stopOpts = new StopCommand();
    cl.addCommand("stop", stopOpts);
    StopAllCommand stopAllOpts = new StopAllCommand();
    cl.addCommand("stopAll", stopAllOpts);
    StopMasterCommand stopMasterOpts = new StopMasterCommand();
    cl.addCommand("stopMaster", stopMasterOpts);
    RandomizeVolumesCommand randomizeVolumesOpts = new RandomizeVolumesCommand();
    cl.addCommand("randomizeVolumes", randomizeVolumesOpts);
    cl.parse(args);
    if (opts.help || cl.getParsedCommand() == null) {
        cl.usage();
        return;
    }
    AccumuloConfiguration siteConf = SiteConfiguration.getInstance();
    // Login as the server on secure HDFS
    if (siteConf.getBoolean(Property.INSTANCE_RPC_SASL_ENABLED)) {
        SecurityUtil.serverLogin(siteConf);
    }
    Instance instance = opts.getInstance();
    ServerConfigurationFactory confFactory = new ServerConfigurationFactory(instance);
    try {
        ClientContext context = new AccumuloServerContext(instance, confFactory);
        int rc = 0;
        if (cl.getParsedCommand().equals("listInstances")) {
            ListInstances.listInstances(instance.getZooKeepers(), listIntancesOpts.printAll, listIntancesOpts.printErrors);
        } else if (cl.getParsedCommand().equals("ping")) {
            if (ping(context, pingCommand.args) != 0)
                rc = 4;
        } else if (cl.getParsedCommand().equals("checkTablets")) {
            System.out.println("\n*** Looking for offline tablets ***\n");
            if (FindOfflineTablets.findOffline(context, checkTabletsCommand.tableName) != 0)
                rc = 5;
            System.out.println("\n*** Looking for missing files ***\n");
            if (checkTabletsCommand.tableName == null) {
                if (RemoveEntriesForMissingFiles.checkAllTables(context, checkTabletsCommand.fixFiles) != 0)
                    rc = 6;
            } else {
                if (RemoveEntriesForMissingFiles.checkTable(context, checkTabletsCommand.tableName, checkTabletsCommand.fixFiles) != 0)
                    rc = 6;
            }
        } else if (cl.getParsedCommand().equals("stop")) {
            stopTabletServer(context, stopOpts.args, opts.force);
        } else if (cl.getParsedCommand().equals("dumpConfig")) {
            printConfig(context, dumpConfigCommand);
        } else if (cl.getParsedCommand().equals("volumes")) {
            ListVolumesUsed.listVolumes(context);
        } else if (cl.getParsedCommand().equals("randomizeVolumes")) {
            rc = RandomizeVolumes.randomize(context.getConnector(), randomizeVolumesOpts.tableName);
        } else {
            everything = cl.getParsedCommand().equals("stopAll");
            if (everything)
                flushAll(context);
            stopServer(context, everything);
        }
        if (rc != 0)
            System.exit(rc);
    } catch (AccumuloException e) {
        log.error("{}", e.getMessage(), e);
        System.exit(1);
    } catch (AccumuloSecurityException e) {
        log.error("{}", e.getMessage(), e);
        System.exit(2);
    } catch (Exception e) {
        log.error("{}", e.getMessage(), e);
        System.exit(3);
    }
}

Example 24 with ClientContext

use of org.apache.accumulo.core.client.impl.ClientContext in project accumulo by apache.

the class VerifyTabletAssignments method main.

public static void main(String[] args) throws Exception {
    Opts opts = new Opts();
    opts.parseArgs(VerifyTabletAssignments.class.getName(), args);
    ClientContext context = new ClientContext(opts.getInstance(), new Credentials(opts.getPrincipal(), opts.getToken()), opts.getClientConfiguration());
    Connector conn = opts.getConnector();
    for (String table : conn.tableOperations().list()) checkTable(context, opts, table, null);
}

Example 25 with ClientContext

use of org.apache.accumulo.core.client.impl.ClientContext in project accumulo by apache.

the class TabletServerResource method getTserverDetails.

/**
 * Generates details for the selected tserver
 *
 * @param tserverAddress
 *          TServer name
 * @return TServer details
 */
@Path("{address}")
@GET
public TabletServerSummary getTserverDetails(@PathParam("address") @NotNull @Pattern(regexp = SERVER_REGEX) String tserverAddress) throws Exception {
    boolean tserverExists = false;
    for (TabletServerStatus ts : Monitor.getMmi().getTServerInfo()) {
        if (tserverAddress.equals(ts.getName())) {
            tserverExists = true;
            break;
        }
    }
    if (!tserverExists) {
        return null;
    }
    double totalElapsedForAll = 0;
    double splitStdDev = 0;
    double minorStdDev = 0;
    double minorQueueStdDev = 0;
    double majorStdDev = 0;
    double majorQueueStdDev = 0;
    double currentMinorAvg = 0;
    double currentMajorAvg = 0;
    double currentMinorStdDev = 0;
    double currentMajorStdDev = 0;
    total = new TabletStats(null, new ActionStats(), new ActionStats(), new ActionStats(), 0, 0, 0, 0);
    HostAndPort address = HostAndPort.fromString(tserverAddress);
    historical = new TabletStats(null, new ActionStats(), new ActionStats(), new ActionStats(), 0, 0, 0, 0);
    List<TabletStats> tsStats = new ArrayList<>();
    try {
        ClientContext context = Monitor.getContext();
        TabletClientService.Client client = ThriftUtil.getClient(new TabletClientService.Client.Factory(), address, context);
        try {
            for (String tableId : Monitor.getMmi().tableMap.keySet()) {
                tsStats.addAll(client.getTabletStats(Tracer.traceInfo(), context.rpcCreds(), tableId));
            }
            historical = client.getHistoricalStats(Tracer.traceInfo(), context.rpcCreds());
        } finally {
            ThriftUtil.returnClient(client);
        }
    } catch (Exception e) {
        return null;
    }
    List<CurrentOperations> currentOps = doCurrentOperations(tsStats);
    if (total.minors.num != 0)
        currentMinorAvg = (long) (total.minors.elapsed / total.minors.num);
    if (total.minors.elapsed != 0 && total.minors.num != 0)
        currentMinorStdDev = stddev(total.minors.elapsed, total.minors.num, total.minors.sumDev);
    if (total.majors.num != 0)
        currentMajorAvg = total.majors.elapsed / total.majors.num;
    if (total.majors.elapsed != 0 && total.majors.num != 0 && total.majors.elapsed > total.majors.num)
        currentMajorStdDev = stddev(total.majors.elapsed, total.majors.num, total.majors.sumDev);
    ActionStatsUpdator.update(total.minors, historical.minors);
    ActionStatsUpdator.update(total.majors, historical.majors);
    totalElapsedForAll += total.majors.elapsed + historical.splits.elapsed + total.minors.elapsed;
    minorStdDev = stddev(total.minors.elapsed, total.minors.num, total.minors.sumDev);
    minorQueueStdDev = stddev(total.minors.queueTime, total.minors.num, total.minors.queueSumDev);
    majorStdDev = stddev(total.majors.elapsed, total.majors.num, total.majors.sumDev);
    majorQueueStdDev = stddev(total.majors.queueTime, total.majors.num, total.majors.queueSumDev);
    splitStdDev = stddev(historical.splits.num, historical.splits.elapsed, historical.splits.sumDev);
    TabletServerDetailInformation details = doDetails(address, tsStats.size());
    List<AllTimeTabletResults> allTime = doAllTimeResults(majorQueueStdDev, minorQueueStdDev, totalElapsedForAll, splitStdDev, majorStdDev, minorStdDev);
    CurrentTabletResults currentRes = doCurrentTabletResults(currentMinorAvg, currentMinorStdDev, currentMajorAvg, currentMajorStdDev);
    TabletServerSummary tserverDetails = new TabletServerSummary(details, allTime, currentRes, currentOps);
    return tserverDetails;
}