use of org.apache.accumulo.core.metadata.schema.TabletMetadata in project accumulo by apache.
the class Gatherer method getFilesGroupedByLocation.
/**
* @param fileSelector
* only returns files that match this predicate
* @return A map of the form : {@code map<tserver location, map<path, list<range>>} . The ranges associated with a file represent the tablets that use the
* file.
*/
private Map<String, Map<String, List<TRowRange>>> getFilesGroupedByLocation(Predicate<String> fileSelector) throws TableNotFoundException, AccumuloException, AccumuloSecurityException {
Iterable<TabletMetadata> tmi = MetadataScanner.builder().from(ctx).overUserTableId(tableId, startRow, endRow).fetchFiles().fetchLocation().fetchLast().fetchPrev().build();
// get a subset of files
Map<String, List<TabletMetadata>> files = new HashMap<>();
for (TabletMetadata tm : tmi) {
for (String file : tm.getFiles()) {
if (fileSelector.test(file)) {
// TODO push this filtering to server side and possibly use batch scanner
files.computeIfAbsent(file, s -> new ArrayList<>()).add(tm);
}
}
}
// group by location, then file
Map<String, Map<String, List<TRowRange>>> locations = new HashMap<>();
List<String> tservers = null;
for (Entry<String, List<TabletMetadata>> entry : files.entrySet()) {
String location = // filter tablets w/o a location
entry.getValue().stream().filter(tm -> tm.getLocation() != null).map(// convert to host:port strings
tm -> tm.getLocation().getHostAndPort().toString()).min(// find minimum host:port
String::compareTo).orElse(// if no locations, then look at last locations
entry.getValue().stream().filter(tm -> tm.getLast() != null).map(// convert to host:port strings
tm -> tm.getLast().getHostAndPort().toString()).min(String::compareTo).orElse(// find minimum last location or return null
null));
if (location == null) {
if (tservers == null) {
tservers = ctx.getConnector().instanceOperations().getTabletServers();
Collections.sort(tservers);
}
// When no location, the approach below will consistently choose the same tserver for the same file (as long as the set of tservers is stable).
int idx = Math.abs(Hashing.murmur3_32().hashString(entry.getKey()).asInt()) % tservers.size();
location = tservers.get(idx);
}
// merge contiguous ranges
List<Range> merged = Range.mergeOverlapping(Lists.transform(entry.getValue(), tm -> tm.getExtent().toDataRange()));
// clip ranges to queried range
List<TRowRange> ranges = merged.stream().map(r -> toClippedExtent(r).toThrift()).collect(Collectors.toList());
locations.computeIfAbsent(location, s -> new HashMap<>()).put(entry.getKey(), ranges);
}
return locations;
}
Aggregations