use of org.apache.accumulo.core.clientImpl.TabletLocator.TabletLocation in project accumulo by apache.
the class MetadataLocationObtainer method getMetadataLocationEntries.
public static TabletLocations getMetadataLocationEntries(SortedMap<Key, Value> entries) {
Text location = null;
Text session = null;
List<TabletLocation> results = new ArrayList<>();
ArrayList<KeyExtent> locationless = new ArrayList<>();
Text lastRowFromKey = new Text();
// text obj below is meant to be reused in loop for efficiency
Text colf = new Text();
Text colq = new Text();
for (Entry<Key, Value> entry : entries.entrySet()) {
Key key = entry.getKey();
Value val = entry.getValue();
if (key.compareRow(lastRowFromKey) != 0) {
location = null;
session = null;
key.getRow(lastRowFromKey);
}
colf = key.getColumnFamily(colf);
colq = key.getColumnQualifier(colq);
// interpret the row id as a key extent
if (colf.equals(CurrentLocationColumnFamily.NAME) || colf.equals(FutureLocationColumnFamily.NAME)) {
if (location != null) {
throw new IllegalStateException("Tablet has multiple locations : " + lastRowFromKey);
}
location = new Text(val.toString());
session = new Text(colq);
} else if (TabletColumnFamily.PREV_ROW_COLUMN.equals(colf, colq)) {
KeyExtent ke = KeyExtent.fromMetaPrevRow(entry);
if (location != null)
results.add(new TabletLocation(ke, location.toString(), session.toString()));
else
locationless.add(ke);
location = null;
}
}
return new TabletLocations(results, locationless);
}
use of org.apache.accumulo.core.clientImpl.TabletLocator.TabletLocation in project accumulo by apache.
the class TabletLocatorImplTest method locateTabletTest.
private void locateTabletTest(TabletLocatorImpl cache, String row, boolean skipRow, KeyExtent expected, String server) throws Exception {
TabletLocation tl = cache.locateTablet(context, new Text(row), skipRow, false);
if (expected == null) {
if (tl != null)
System.out.println("tl = " + tl);
assertNull(tl);
} else {
assertNotNull(tl);
assertEquals(server, tl.tablet_location);
assertEquals(expected, tl.tablet_extent);
}
}
use of org.apache.accumulo.core.clientImpl.TabletLocator.TabletLocation in project accumulo by apache.
the class TabletLocatorImplTest method runTest.
static void runTest(TreeMap<Text, TabletLocation> mc, KeyExtent remove, Set<KeyExtent> expected) {
// copy so same metaCache can be used for multiple test
mc = new TreeMap<>(mc);
TabletLocatorImpl.removeOverlapping(mc, remove);
HashSet<KeyExtent> eic = new HashSet<>();
for (TabletLocation tl : mc.values()) {
eic.add(tl.tablet_extent);
}
assertEquals(expected, eic);
}
use of org.apache.accumulo.core.clientImpl.TabletLocator.TabletLocation in project accumulo by apache.
the class BulkImporter method importFiles.
public AssignmentStats importFiles(List<String> files) {
int numThreads = context.getConfiguration().getCount(Property.TSERV_BULK_PROCESS_THREADS);
int numAssignThreads = context.getConfiguration().getCount(Property.TSERV_BULK_ASSIGNMENT_THREADS);
timer = new StopWatch<>(Timers.class);
timer.start(Timers.TOTAL);
final VolumeManager fs = context.getVolumeManager();
Set<Path> paths = new HashSet<>();
for (String file : files) {
paths.add(new Path(file));
}
AssignmentStats assignmentStats = new AssignmentStats(paths.size());
final Map<Path, List<KeyExtent>> completeFailures = Collections.synchronizedSortedMap(new TreeMap<>());
ClientService.Client client = null;
final TabletLocator locator = TabletLocator.getLocator(context, TableId.of(tableId));
try {
final Map<Path, List<TabletLocation>> assignments = Collections.synchronizedSortedMap(new TreeMap<>());
timer.start(Timers.EXAMINE_MAP_FILES);
ExecutorService threadPool = ThreadPools.createFixedThreadPool(numThreads, "findOverlapping", false);
for (Path path : paths) {
final Path mapFile = path;
Runnable getAssignments = new Runnable() {
@Override
public void run() {
List<TabletLocation> tabletsToAssignMapFileTo = Collections.emptyList();
try {
tabletsToAssignMapFileTo = findOverlappingTablets(context, fs, locator, mapFile);
} catch (Exception ex) {
log.warn("Unable to find tablets that overlap file " + mapFile, ex);
}
log.debug("Map file {} found to overlap {} tablets", mapFile, tabletsToAssignMapFileTo.size());
if (tabletsToAssignMapFileTo.isEmpty()) {
List<KeyExtent> empty = Collections.emptyList();
completeFailures.put(mapFile, empty);
} else
assignments.put(mapFile, tabletsToAssignMapFileTo);
}
};
threadPool.submit(getAssignments);
}
threadPool.shutdown();
while (!threadPool.isTerminated()) {
try {
threadPool.awaitTermination(60, TimeUnit.SECONDS);
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
}
timer.stop(Timers.EXAMINE_MAP_FILES);
assignmentStats.attemptingAssignments(assignments);
Map<Path, List<KeyExtent>> assignmentFailures = assignMapFiles(fs, assignments, paths, numAssignThreads, numThreads);
assignmentStats.assignmentsFailed(assignmentFailures);
Map<Path, Integer> failureCount = new TreeMap<>();
for (Entry<Path, List<KeyExtent>> entry : assignmentFailures.entrySet()) failureCount.put(entry.getKey(), 1);
long sleepTime = 2_000;
while (!assignmentFailures.isEmpty()) {
sleepTime = Math.min(sleepTime * 2, MINUTES.toMillis(1));
locator.invalidateCache();
// assumption about assignment failures is that it caused by a split
// happening or a missing location
//
// for splits we need to find children key extents that cover the
// same key range and are contiguous (no holes, no overlap)
timer.start(Timers.SLEEP);
sleepUninterruptibly(sleepTime, TimeUnit.MILLISECONDS);
timer.stop(Timers.SLEEP);
log.debug("Trying to assign {} map files that previously failed on some key extents", assignmentFailures.size());
assignments.clear();
// assign to
for (Entry<Path, List<KeyExtent>> entry : assignmentFailures.entrySet()) {
Iterator<KeyExtent> keListIter = entry.getValue().iterator();
List<TabletLocation> tabletsToAssignMapFileTo = new ArrayList<>();
while (keListIter.hasNext()) {
KeyExtent ke = keListIter.next();
timer.start(Timers.QUERY_METADATA);
try {
tabletsToAssignMapFileTo.addAll(findOverlappingTablets(context, fs, locator, entry.getKey(), ke));
keListIter.remove();
} catch (Exception ex) {
log.warn("Exception finding overlapping tablets, will retry tablet " + ke, ex);
}
timer.stop(Timers.QUERY_METADATA);
}
if (!tabletsToAssignMapFileTo.isEmpty())
assignments.put(entry.getKey(), tabletsToAssignMapFileTo);
}
assignmentStats.attemptingAssignments(assignments);
Map<Path, List<KeyExtent>> assignmentFailures2 = assignMapFiles(fs, assignments, paths, numAssignThreads, numThreads);
assignmentStats.assignmentsFailed(assignmentFailures2);
// merge assignmentFailures2 into assignmentFailures
for (Entry<Path, List<KeyExtent>> entry : assignmentFailures2.entrySet()) {
assignmentFailures.get(entry.getKey()).addAll(entry.getValue());
Integer fc = failureCount.get(entry.getKey());
if (fc == null)
fc = 0;
failureCount.put(entry.getKey(), fc + 1);
}
// remove map files that have no more key extents to assign
assignmentFailures.values().removeIf(List::isEmpty);
Set<Entry<Path, Integer>> failureIter = failureCount.entrySet();
for (Entry<Path, Integer> entry : failureIter) {
int retries = context.getConfiguration().getCount(Property.TSERV_BULK_RETRY);
if (entry.getValue() > retries && assignmentFailures.get(entry.getKey()) != null) {
log.error("Map file {} failed more than {} times, giving up.", entry.getKey(), retries);
completeFailures.put(entry.getKey(), assignmentFailures.get(entry.getKey()));
assignmentFailures.remove(entry.getKey());
}
}
}
assignmentStats.assignmentsAbandoned(completeFailures);
Set<Path> failedFailures = processFailures(completeFailures);
assignmentStats.unrecoveredMapFiles(failedFailures);
timer.stop(Timers.TOTAL);
printReport(paths);
return assignmentStats;
} finally {
if (client != null) {
ServerClient.close(client, context);
}
}
}
use of org.apache.accumulo.core.clientImpl.TabletLocator.TabletLocation in project accumulo by apache.
the class BulkImporter method estimateSizes.
private Map<Path, List<AssignmentInfo>> estimateSizes(final VolumeManager vm, Map<Path, List<TabletLocation>> assignments, Collection<Path> paths, int numThreads) {
long t1 = System.currentTimeMillis();
final Map<Path, Long> mapFileSizes = new TreeMap<>();
try {
for (Path path : paths) {
FileSystem fs = vm.getFileSystemByPath(path);
mapFileSizes.put(path, fs.getContentSummary(path).getLength());
}
} catch (IOException e) {
log.error("Failed to get map files in for {}: {}", paths, e.getMessage(), e);
throw new RuntimeException(e);
}
final Map<Path, List<AssignmentInfo>> ais = Collections.synchronizedMap(new TreeMap<>());
ExecutorService threadPool = ThreadPools.createFixedThreadPool(numThreads, "estimateSizes", false);
for (final Entry<Path, List<TabletLocation>> entry : assignments.entrySet()) {
if (entry.getValue().size() == 1) {
TabletLocation tabletLocation = entry.getValue().get(0);
// if the tablet completely contains the map file, there is no
// need to estimate its
// size
ais.put(entry.getKey(), Collections.singletonList(new AssignmentInfo(tabletLocation.tablet_extent, mapFileSizes.get(entry.getKey()))));
continue;
}
Runnable estimationTask = new Runnable() {
@Override
public void run() {
Map<KeyExtent, Long> estimatedSizes = null;
try {
estimatedSizes = FileUtil.estimateSizes(context, entry.getKey(), mapFileSizes.get(entry.getKey()), extentsOf(entry.getValue()));
} catch (IOException e) {
log.warn("Failed to estimate map file sizes {}", e.getMessage());
}
if (estimatedSizes == null) {
// estimation failed, do a simple estimation
estimatedSizes = new TreeMap<>();
long estSize = (long) (mapFileSizes.get(entry.getKey()) / (double) entry.getValue().size());
for (TabletLocation tl : entry.getValue()) estimatedSizes.put(tl.tablet_extent, estSize);
}
List<AssignmentInfo> assignmentInfoList = new ArrayList<>(estimatedSizes.size());
for (Entry<KeyExtent, Long> entry2 : estimatedSizes.entrySet()) assignmentInfoList.add(new AssignmentInfo(entry2.getKey(), entry2.getValue()));
ais.put(entry.getKey(), assignmentInfoList);
}
};
threadPool.submit(estimationTask);
}
threadPool.shutdown();
while (!threadPool.isTerminated()) {
try {
threadPool.awaitTermination(60, TimeUnit.SECONDS);
} catch (InterruptedException e) {
log.error("Encountered InterruptedException while waiting for the threadPool to terminate.", e);
throw new RuntimeException(e);
}
}
long t2 = System.currentTimeMillis();
log.debug(String.format("Estimated map files sizes in %6.2f secs", (t2 - t1) / 1000.0));
return ais;
}
Aggregations