Examples with LOCATION org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.LOCATION used on opensource projects

Example 1 with LOCATION

use of org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.LOCATION in project accumulo by apache.

the class Gatherer method getFilesGroupedByLocation.

/**
 * @param fileSelector
 *          only returns files that match this predicate
 * @return A map of the form : {@code map<tserver location, map<path, list<range>>} . The ranges
 *         associated with a file represent the tablets that use the file.
 */
private Map<String, Map<TabletFile, List<TRowRange>>> getFilesGroupedByLocation(Predicate<TabletFile> fileSelector) {
    Iterable<TabletMetadata> tmi = TabletsMetadata.builder(ctx).forTable(tableId).overlapping(startRow, endRow).fetch(FILES, LOCATION, LAST, PREV_ROW).build();
    // get a subset of files
    Map<TabletFile, List<TabletMetadata>> files = new HashMap<>();
    for (TabletMetadata tm : tmi) {
        for (TabletFile file : tm.getFiles()) {
            if (fileSelector.test(file)) {
                // TODO push this filtering to server side and possibly use batch scanner
                files.computeIfAbsent(file, s -> new ArrayList<>()).add(tm);
            }
        }
    }
    // group by location, then file
    Map<String, Map<TabletFile, List<TRowRange>>> locations = new HashMap<>();
    List<String> tservers = null;
    for (Entry<TabletFile, List<TabletMetadata>> entry : files.entrySet()) {
        String location = // filter
        entry.getValue().stream().filter(tm -> tm.getLocation() != null).map(// convert to host:port strings
        tm -> tm.getLocation().getHostPort()).min(// find minimum host:port
        String::compareTo).orElse(// if no locations,
        entry.getValue().stream().filter(tm -> tm.getLast() != null).map(// convert to host:port strings
        tm -> tm.getLast().getHostPort()).min(String::compareTo).orElse(// find minimum last location or return null
        null));
        if (location == null) {
            if (tservers == null) {
                tservers = ctx.instanceOperations().getTabletServers();
                Collections.sort(tservers);
            }
            // When no location, the approach below will consistently choose the same tserver for the
            // same file (as long as the set of tservers is stable).
            int idx = Math.abs(Hashing.murmur3_32_fixed().hashString(entry.getKey().getPathStr(), UTF_8).asInt()) % tservers.size();
            location = tservers.get(idx);
        }
        // merge contiguous ranges
        List<Range> merged = Range.mergeOverlapping(Lists.transform(entry.getValue(), tm -> tm.getExtent().toDataRange()));
        List<TRowRange> ranges = // clip
        merged.stream().map(r -> toClippedExtent(r).toThrift()).collect(Collectors.toList());
        // ranges
        // to
        // queried
        // range
        locations.computeIfAbsent(location, s -> new HashMap<>()).put(entry.getKey(), ranges);
    }
    return locations;
}