use of org.apache.accumulo.core.clientImpl.TabletLocator in project accumulo by apache.
the class AbstractInputFormat method getSplits.
/**
* Gets the splits of the tables that have been set on the job by reading the metadata table for
* the specified ranges.
*
* @return the splits from the tables based on the ranges.
* @throws java.io.IOException
* if a table set on the job doesn't exist or an error occurs initializing the tablet
* locator
*/
@Override
public List<InputSplit> getSplits(JobContext job) throws IOException {
Level logLevel = getLogLevel(job);
log.setLevel(logLevel);
validateOptions(job);
LinkedList<InputSplit> splits = new LinkedList<>();
Map<String, InputTableConfig> tableConfigs = getInputTableConfigs(job);
for (Map.Entry<String, InputTableConfig> tableConfigEntry : tableConfigs.entrySet()) {
String tableName = tableConfigEntry.getKey();
InputTableConfig tableConfig = tableConfigEntry.getValue();
ClientContext client;
try {
client = InputConfigurator.client(CLASS, job.getConfiguration());
} catch (AccumuloException | AccumuloSecurityException e) {
throw new IOException(e);
}
TableId tableId;
// resolve table name to id once, and use id from this point forward
try {
tableId = client.getTableId(tableName);
} catch (TableNotFoundException e) {
throw new IOException(e);
}
boolean batchScan = InputConfigurator.isBatchScan(CLASS, job.getConfiguration());
boolean supportBatchScan = !(tableConfig.isOfflineScan() || tableConfig.shouldUseIsolatedScanners() || tableConfig.shouldUseLocalIterators());
if (batchScan && !supportBatchScan)
throw new IllegalArgumentException("BatchScanner optimization not available for offline" + " scan, isolated, or local iterators");
boolean autoAdjust = tableConfig.shouldAutoAdjustRanges();
if (batchScan && !autoAdjust)
throw new IllegalArgumentException("AutoAdjustRanges must be enabled when using BatchScanner optimization");
List<Range> ranges = autoAdjust ? Range.mergeOverlapping(tableConfig.getRanges()) : tableConfig.getRanges();
if (ranges.isEmpty()) {
ranges = new ArrayList<>(1);
ranges.add(new Range());
}
// get the metadata information for these ranges
Map<String, Map<KeyExtent, List<Range>>> binnedRanges = new HashMap<>();
TabletLocator tl;
try {
if (tableConfig.isOfflineScan()) {
binnedRanges = binOfflineTable(job, tableId, ranges);
while (binnedRanges == null) {
// Some tablets were still online, try again
// sleep randomly between 100 and 200 ms
sleepUninterruptibly(100 + random.nextInt(100), TimeUnit.MILLISECONDS);
binnedRanges = binOfflineTable(job, tableId, ranges);
}
} else {
tl = TabletLocator.getLocator(client, tableId);
// its possible that the cache could contain complete, but old information about a
// tables tablets... so clear it
tl.invalidateCache();
while (!tl.binRanges(client, ranges, binnedRanges).isEmpty()) {
client.requireNotDeleted(tableId);
client.requireNotOffline(tableId, tableName);
binnedRanges.clear();
log.warn("Unable to locate bins for specified ranges. Retrying.");
// sleep randomly between 100 and 200 ms
sleepUninterruptibly(100 + random.nextInt(100), TimeUnit.MILLISECONDS);
tl.invalidateCache();
}
}
} catch (Exception e) {
throw new IOException(e);
}
// all of this code will add either range per each locations or split ranges and add
// range-location split
// Map from Range to Array of Locations, we only use this if we're don't split
HashMap<Range, ArrayList<String>> splitsToAdd = null;
if (!autoAdjust)
splitsToAdd = new HashMap<>();
HashMap<String, String> hostNameCache = new HashMap<>();
for (Map.Entry<String, Map<KeyExtent, List<Range>>> tserverBin : binnedRanges.entrySet()) {
String ip = tserverBin.getKey().split(":", 2)[0];
String location = hostNameCache.get(ip);
if (location == null) {
InetAddress inetAddress = InetAddress.getByName(ip);
location = inetAddress.getCanonicalHostName();
hostNameCache.put(ip, location);
}
for (Map.Entry<KeyExtent, List<Range>> extentRanges : tserverBin.getValue().entrySet()) {
Range ke = extentRanges.getKey().toDataRange();
if (batchScan) {
// group ranges by tablet to be read by a BatchScanner
ArrayList<Range> clippedRanges = new ArrayList<>();
for (Range r : extentRanges.getValue()) clippedRanges.add(ke.clip(r));
org.apache.accumulo.core.clientImpl.mapreduce.BatchInputSplit split = new org.apache.accumulo.core.clientImpl.mapreduce.BatchInputSplit(tableName, tableId, clippedRanges, new String[] { location });
org.apache.accumulo.core.clientImpl.mapreduce.SplitUtils.updateSplit(split, tableConfig, logLevel);
splits.add(split);
} else {
// not grouping by tablet
for (Range r : extentRanges.getValue()) {
if (autoAdjust) {
// divide ranges into smaller ranges, based on the tablets
RangeInputSplit split = new RangeInputSplit(tableName, tableId.canonical(), ke.clip(r), new String[] { location });
org.apache.accumulo.core.clientImpl.mapreduce.SplitUtils.updateSplit(split, tableConfig, logLevel);
split.setOffline(tableConfig.isOfflineScan());
split.setIsolatedScan(tableConfig.shouldUseIsolatedScanners());
split.setUsesLocalIterators(tableConfig.shouldUseLocalIterators());
splits.add(split);
} else {
// don't divide ranges
ArrayList<String> locations = splitsToAdd.get(r);
if (locations == null)
locations = new ArrayList<>(1);
locations.add(location);
splitsToAdd.put(r, locations);
}
}
}
}
}
if (!autoAdjust)
for (Map.Entry<Range, ArrayList<String>> entry : splitsToAdd.entrySet()) {
RangeInputSplit split = new RangeInputSplit(tableName, tableId.canonical(), entry.getKey(), entry.getValue().toArray(new String[0]));
org.apache.accumulo.core.clientImpl.mapreduce.SplitUtils.updateSplit(split, tableConfig, logLevel);
split.setOffline(tableConfig.isOfflineScan());
split.setIsolatedScan(tableConfig.shouldUseIsolatedScanners());
split.setUsesLocalIterators(tableConfig.shouldUseLocalIterators());
splits.add(split);
}
}
return splits;
}
use of org.apache.accumulo.core.clientImpl.TabletLocator in project accumulo by apache.
the class MetadataLocationObtainer method lookupTablets.
@Override
public List<TabletLocation> lookupTablets(ClientContext context, String tserver, Map<KeyExtent, List<Range>> tabletsRanges, TabletLocator parent) throws AccumuloSecurityException, AccumuloException {
final TreeMap<Key, Value> results = new TreeMap<>();
ResultReceiver rr = entries -> {
for (Entry<Key, Value> entry : entries) {
try {
results.putAll(WholeRowIterator.decodeRow(entry.getKey(), entry.getValue()));
} catch (IOException e) {
throw new RuntimeException(e);
}
}
};
ScannerOptions opts = null;
try (SettableScannerOptions unsetOpts = new SettableScannerOptions()) {
opts = unsetOpts.setColumns(locCols);
}
Map<KeyExtent, List<Range>> unscanned = new HashMap<>();
Map<KeyExtent, List<Range>> failures = new HashMap<>();
try {
TabletServerBatchReaderIterator.doLookup(context, tserver, tabletsRanges, failures, unscanned, rr, columns, opts, Authorizations.EMPTY);
if (!failures.isEmpty()) {
// invalidate extents in parents cache
if (log.isTraceEnabled())
log.trace("lookupTablets failed for {} extents", failures.size());
parent.invalidateCache(failures.keySet());
}
} catch (IOException e) {
log.trace("lookupTablets failed server={}", tserver, e);
parent.invalidateCache(context, tserver);
} catch (AccumuloServerException e) {
log.trace("lookupTablets failed server={}", tserver, e);
throw e;
}
return MetadataLocationObtainer.getMetadataLocationEntries(results).getLocations();
}
use of org.apache.accumulo.core.clientImpl.TabletLocator in project accumulo by apache.
the class BulkFailureIT method getClient.
protected static TabletClientService.Iface getClient(ClientContext context, KeyExtent extent) throws AccumuloException, AccumuloSecurityException, TableNotFoundException, TTransportException {
TabletLocator locator = TabletLocator.getLocator(context, extent.tableId());
locator.invalidateCache(extent);
HostAndPort location = HostAndPort.fromString(locator.locateTablet(context, new Text(""), false, true).tablet_location);
long timeInMillis = context.getConfiguration().getTimeInMillis(Property.TSERV_BULK_TIMEOUT);
TabletClientService.Iface client = ThriftUtil.getTServerClient(location, context, timeInMillis);
return client;
}
use of org.apache.accumulo.core.clientImpl.TabletLocator in project accumulo by apache.
the class AccumuloRecordReader method getSplits.
public static List<InputSplit> getSplits(JobContext context, Class<?> callingClass) throws IOException {
validateOptions(context, callingClass);
LinkedList<InputSplit> splits = new LinkedList<>();
try (AccumuloClient client = createClient(context, callingClass);
var clientContext = ((ClientContext) client)) {
Map<String, InputTableConfig> tableConfigs = InputConfigurator.getInputTableConfigs(callingClass, context.getConfiguration());
for (Map.Entry<String, InputTableConfig> tableConfigEntry : tableConfigs.entrySet()) {
String tableName = tableConfigEntry.getKey();
InputTableConfig tableConfig = tableConfigEntry.getValue();
TableId tableId;
// resolve table name to id once, and use id from this point forward
try {
tableId = clientContext.getTableId(tableName);
} catch (TableNotFoundException e) {
throw new IOException(e);
}
boolean batchScan = InputConfigurator.isBatchScan(callingClass, context.getConfiguration());
boolean supportBatchScan = !(tableConfig.isOfflineScan() || tableConfig.shouldUseIsolatedScanners() || tableConfig.shouldUseLocalIterators());
if (batchScan && !supportBatchScan)
throw new IllegalArgumentException("BatchScanner optimization not available for offline" + " scan, isolated, or local iterators");
boolean autoAdjust = tableConfig.shouldAutoAdjustRanges();
if (batchScan && !autoAdjust)
throw new IllegalArgumentException("AutoAdjustRanges must be enabled when using BatchScanner optimization");
List<Range> ranges = autoAdjust ? Range.mergeOverlapping(tableConfig.getRanges()) : tableConfig.getRanges();
if (ranges.isEmpty()) {
ranges = new ArrayList<>(1);
ranges.add(new Range());
}
// get the metadata information for these ranges
Map<String, Map<KeyExtent, List<Range>>> binnedRanges = new HashMap<>();
TabletLocator tl;
try {
if (tableConfig.isOfflineScan()) {
binnedRanges = binOfflineTable(context, tableId, ranges, callingClass);
while (binnedRanges == null) {
// Some tablets were still online, try again
// sleep randomly between 100 and 200 ms
sleepUninterruptibly(100 + random.nextInt(100), TimeUnit.MILLISECONDS);
binnedRanges = binOfflineTable(context, tableId, ranges, callingClass);
}
} else {
tl = InputConfigurator.getTabletLocator(callingClass, context.getConfiguration(), tableId);
// its possible that the cache could contain complete, but old information about a
// tables tablets... so clear it
tl.invalidateCache();
while (!tl.binRanges(clientContext, ranges, binnedRanges).isEmpty()) {
clientContext.requireNotDeleted(tableId);
clientContext.requireNotOffline(tableId, tableName);
binnedRanges.clear();
log.warn("Unable to locate bins for specified ranges. Retrying.");
// sleep randomly between 100 and 200 ms
sleepUninterruptibly(100 + random.nextInt(100), TimeUnit.MILLISECONDS);
tl.invalidateCache();
}
}
} catch (TableOfflineException | TableNotFoundException | AccumuloException | AccumuloSecurityException e) {
throw new IOException(e);
}
// all of this code will add either range per each locations or split ranges and add
// range-location split
// Map from Range to Array of Locations, we only use this if we're don't split
HashMap<Range, ArrayList<String>> splitsToAdd = null;
if (!autoAdjust)
splitsToAdd = new HashMap<>();
HashMap<String, String> hostNameCache = new HashMap<>();
for (Map.Entry<String, Map<KeyExtent, List<Range>>> tserverBin : binnedRanges.entrySet()) {
String ip = tserverBin.getKey().split(":", 2)[0];
String location = hostNameCache.get(ip);
if (location == null) {
InetAddress inetAddress = InetAddress.getByName(ip);
location = inetAddress.getCanonicalHostName();
hostNameCache.put(ip, location);
}
for (Map.Entry<KeyExtent, List<Range>> extentRanges : tserverBin.getValue().entrySet()) {
Range ke = extentRanges.getKey().toDataRange();
if (batchScan) {
// group ranges by tablet to be read by a BatchScanner
ArrayList<Range> clippedRanges = new ArrayList<>();
for (Range r : extentRanges.getValue()) clippedRanges.add(ke.clip(r));
BatchInputSplit split = new BatchInputSplit(tableName, tableId, clippedRanges, new String[] { location });
SplitUtils.updateSplit(split, tableConfig);
splits.add(split);
} else {
// not grouping by tablet
for (Range r : extentRanges.getValue()) {
if (autoAdjust) {
// divide ranges into smaller ranges, based on the tablets
RangeInputSplit split = new RangeInputSplit(tableName, tableId.canonical(), ke.clip(r), new String[] { location });
SplitUtils.updateSplit(split, tableConfig);
split.setOffline(tableConfig.isOfflineScan());
split.setIsolatedScan(tableConfig.shouldUseIsolatedScanners());
split.setUsesLocalIterators(tableConfig.shouldUseLocalIterators());
splits.add(split);
} else {
// don't divide ranges
ArrayList<String> locations = splitsToAdd.get(r);
if (locations == null)
locations = new ArrayList<>(1);
locations.add(location);
splitsToAdd.put(r, locations);
}
}
}
}
}
if (!autoAdjust)
for (Map.Entry<Range, ArrayList<String>> entry : splitsToAdd.entrySet()) {
RangeInputSplit split = new RangeInputSplit(tableName, tableId.canonical(), entry.getKey(), entry.getValue().toArray(new String[0]));
SplitUtils.updateSplit(split, tableConfig);
split.setOffline(tableConfig.isOfflineScan());
split.setIsolatedScan(tableConfig.shouldUseIsolatedScanners());
split.setUsesLocalIterators(tableConfig.shouldUseLocalIterators());
splits.add(split);
}
}
}
return splits;
}
use of org.apache.accumulo.core.clientImpl.TabletLocator in project accumulo by apache.
the class SuspendedTabletsIT method shutdownAndResumeTserver.
@Test
public void shutdownAndResumeTserver() throws Exception {
// Run the test body. When we get to the point where we need tservers to go away, stop them via
// a clean shutdown.
suspensionTestBody((ctx, locs, count) -> {
Set<TServerInstance> tserverSet = new HashSet<>();
Set<TServerInstance> metadataServerSet = new HashSet<>();
TabletLocator tl = TabletLocator.getLocator(ctx, MetadataTable.ID);
for (TabletLocationState tls : locs.locationStates.values()) {
if (tls.current != null) {
// add to set of all servers
tserverSet.add(tls.current);
// get server that the current tablets metadata is on
TabletLocator.TabletLocation tab = tl.locateTablet(ctx, tls.extent.toMetaRow(), false, false);
// add it to the set of servers with metadata
metadataServerSet.add(new TServerInstance(tab.tablet_location, Long.valueOf(tab.tablet_session, 16)));
}
}
// remove servers with metadata on them from the list of servers to be shutdown
assertEquals("Expecting a single tServer in metadataServerSet", 1, metadataServerSet.size());
tserverSet.removeAll(metadataServerSet);
assertEquals("Expecting " + (TSERVERS - 1) + " tServers in shutdown-list", TSERVERS - 1, tserverSet.size());
List<TServerInstance> tserversList = new ArrayList<>(tserverSet);
Collections.shuffle(tserversList, random);
for (int i1 = 0; i1 < count; ++i1) {
final String tserverName = tserversList.get(i1).getHostPortSession();
ManagerClient.executeVoid(ctx, client -> {
log.info("Sending shutdown command to {} via ManagerClientService", tserverName);
client.shutdownTabletServer(null, ctx.rpcCreds(), tserverName, false);
});
}
log.info("Waiting for tserver process{} to die", count == 1 ? "" : "es");
for (int i2 = 0; i2 < 10; ++i2) {
List<ProcessReference> deadProcs = new ArrayList<>();
for (ProcessReference pr1 : getCluster().getProcesses().get(ServerType.TABLET_SERVER)) {
Process p = pr1.getProcess();
if (!p.isAlive()) {
deadProcs.add(pr1);
}
}
for (ProcessReference pr2 : deadProcs) {
log.info("Process {} is dead, informing cluster control about this", pr2.getProcess());
getCluster().getClusterControl().killProcess(ServerType.TABLET_SERVER, pr2);
--count;
}
if (count == 0) {
return;
} else {
Thread.sleep(SECONDS.toMillis(2));
}
}
throw new IllegalStateException("Tablet servers didn't die!");
});
}
Aggregations