Examples with KeyExtent org.apache.accumulo.core.dataImpl.KeyExtent used on opensource projects

Example 31 with KeyExtent

use of org.apache.accumulo.core.dataImpl.KeyExtent in project accumulo by apache.

the class GroupBalancer method populateMigrations.

private void populateMigrations(Set<TServerInstance> current, List<TabletMigration> migrationsOut, Moves moves) {
    if (moves.size() == 0) {
        return;
    }
    Function<KeyExtent, String> partitioner = getPartitioner();
    for (var tablet : getLocationProvider().entrySet()) {
        String group = partitioner.apply(tablet.getKey());
        var loc = tablet.getValue();
        if (loc == null || !current.contains(loc)) {
            migrationsOut.clear();
            return;
        }
        TServerInstance dest = moves.removeMove(loc, group);
        if (dest != null) {
            migrationsOut.add(new TabletMigration(tablet.getKey(), loc, dest));
            if (moves.size() == 0) {
                break;
            }
        }
    }
}

Example 32 with KeyExtent

use of org.apache.accumulo.core.dataImpl.KeyExtent in project accumulo by apache.

the class GroupBalancer method getAssignments.

@Override
public void getAssignments(SortedMap<TServerInstance, TabletServerStatus> current, Map<KeyExtent, TServerInstance> unassigned, Map<KeyExtent, TServerInstance> assignments) {
    if (current.isEmpty()) {
        return;
    }
    Function<KeyExtent, String> partitioner = getPartitioner();
    List<ComparablePair<String, KeyExtent>> tabletsByGroup = new ArrayList<>();
    for (Entry<KeyExtent, TServerInstance> entry : unassigned.entrySet()) {
        TServerInstance last = entry.getValue();
        if (last != null) {
            // Maintain locality
            String fakeSessionID = " ";
            TServerInstance simple = new TServerInstance(last.getHostAndPort(), fakeSessionID);
            Iterator<TServerInstance> find = current.tailMap(simple).keySet().iterator();
            if (find.hasNext()) {
                TServerInstance tserver = find.next();
                if (tserver.getHost().equals(last.getHost())) {
                    assignments.put(entry.getKey(), tserver);
                    continue;
                }
            }
        }
        tabletsByGroup.add(new ComparablePair<>(partitioner.apply(entry.getKey()), entry.getKey()));
    }
    Collections.sort(tabletsByGroup);
    Iterator<TServerInstance> tserverIter = Iterators.cycle(current.keySet());
    for (ComparablePair<String, KeyExtent> pair : tabletsByGroup) {
        KeyExtent ke = pair.getSecond();
        assignments.put(ke, tserverIter.next());
    }
}

Example 33 with KeyExtent

use of org.apache.accumulo.core.dataImpl.KeyExtent in project accumulo by apache.

the class GroupBalancer method balance.

@Override
public long balance(SortedMap<TServerInstance, TabletServerStatus> current, Set<KeyExtent> migrations, List<TabletMigration> migrationsOut) {
    if (!shouldBalance(current, migrations)) {
        return 5000;
    }
    if (System.currentTimeMillis() - lastRun < getWaitTime()) {
        return 5000;
    }
    MapCounter<String> groupCounts = new MapCounter<>();
    Map<TServerInstance, TserverGroupInfo> tservers = new HashMap<>();
    for (TServerInstance tsi : current.keySet()) {
        tservers.put(tsi, new TserverGroupInfo(tsi));
    }
    Function<KeyExtent, String> partitioner = getPartitioner();
    // collect stats about current state
    for (var tablet : getLocationProvider().entrySet()) {
        String group = partitioner.apply(tablet.getKey());
        var loc = tablet.getValue();
        if (loc == null || !tservers.containsKey(loc)) {
            return 5000;
        }
        groupCounts.increment(group, 1);
        TserverGroupInfo tgi = tservers.get(loc);
        tgi.addGroup(group);
    }
    Map<String, Integer> expectedCounts = new HashMap<>();
    int totalExtra = 0;
    for (String group : groupCounts.keySet()) {
        int groupCount = groupCounts.getInt(group);
        totalExtra += groupCount % current.size();
        expectedCounts.put(group, (groupCount / current.size()));
    }
    // The number of extra tablets from all groups that each tserver must have.
    int expectedExtra = totalExtra / current.size();
    int maxExtraGroups = expectedExtra + 1;
    expectedCounts = Collections.unmodifiableMap(expectedCounts);
    tservers = Collections.unmodifiableMap(tservers);
    for (TserverGroupInfo tgi : tservers.values()) {
        tgi.finishedAdding(expectedCounts);
    }
    Moves moves = new Moves();
    // The order of the following steps is important, because as ordered each step should not move
    // any tablets moved by a previous step.
    balanceExpected(tservers, moves);
    if (moves.size() < getMaxMigrations()) {
        balanceExtraExpected(tservers, expectedExtra, moves);
        if (moves.size() < getMaxMigrations()) {
            boolean cont = balanceExtraMultiple(tservers, maxExtraGroups, moves);
            if (cont && moves.size() < getMaxMigrations()) {
                balanceExtraExtra(tservers, maxExtraGroups, moves);
            }
        }
    }
    populateMigrations(tservers.keySet(), migrationsOut, moves);
    lastRun = System.currentTimeMillis();
    return 5000;
}

Example 34 with KeyExtent

use of org.apache.accumulo.core.dataImpl.KeyExtent in project accumulo by apache.

the class RegexGroupBalancer method getPartitioner.

@Override
protected Function<KeyExtent, String> getPartitioner() {
    Map<String, String> customProps = context.getTableConfiguration(tableId).getAllPropertiesWithPrefix(Property.TABLE_ARBITRARY_PROP_PREFIX);
    String regex = customProps.get(REGEX_PROPERTY);
    final String defaultGroup = customProps.get(DEFAUT_GROUP_PROPERTY);
    final Pattern pattern = Pattern.compile(regex);
    return new Function<>() {

        @Override
        public String apply(KeyExtent input) {
            Text er = input.endRow();
            if (er == null) {
                return defaultGroup;
            }
            Matcher matcher = pattern.matcher(er.toString());
            if (matcher.matches() && matcher.groupCount() == 1) {
                return matcher.group(1);
            }
            return defaultGroup;
        }
    };
}

Example 35 with KeyExtent

use of org.apache.accumulo.core.dataImpl.KeyExtent in project accumulo by apache.

the class MetadataConstraints method check.

@Override
public List<Short> check(Environment env, Mutation mutation) {
    final ServerContext context = ((SystemEnvironment) env).getServerContext();
    ArrayList<Short> violations = null;
    Collection<ColumnUpdate> colUpdates = mutation.getUpdates();
    // check the row, it should contains at least one ; or end with <
    boolean containsSemiC = false;
    byte[] row = mutation.getRow();
    // always allow rows that fall within reserved areas
    if (row.length > 0 && row[0] == '~')
        return null;
    if (row.length > 2 && row[0] == '!' && row[1] == '!' && row[2] == '~')
        return null;
    for (byte b : row) {
        if (b == ';') {
            containsSemiC = true;
        }
        if (b == ';' || b == '<')
            break;
        if (!validTableNameChars[0xff & b]) {
            violations = addIfNotPresent(violations, 4);
        }
    }
    if (containsSemiC) {
        if (row.length == 0) {
            violations = addIfNotPresent(violations, 4);
        }
    } else {
        // see if last row char is <
        if (row.length == 0 || row[row.length - 1] != '<') {
            violations = addIfNotPresent(violations, 4);
        }
    }
    if (row.length > 0 && row[0] == '!') {
        if (row.length < 3 || row[1] != '0' || (row[2] != '<' && row[2] != ';')) {
            violations = addIfNotPresent(violations, 4);
        }
    }
    // ensure row is not less than Constants.METADATA_TABLE_ID
    if (new Text(row).compareTo(new Text(MetadataTable.ID.canonical())) < 0) {
        violations = addViolation(violations, 5);
    }
    boolean checkedBulk = false;
    for (ColumnUpdate columnUpdate : colUpdates) {
        Text columnFamily = new Text(columnUpdate.getColumnFamily());
        if (columnUpdate.isDeleted()) {
            if (!isValidColumn(columnUpdate)) {
                violations = addViolation(violations, 2);
            }
            continue;
        }
        if (columnUpdate.getValue().length == 0 && !columnFamily.equals(ScanFileColumnFamily.NAME)) {
            violations = addViolation(violations, 6);
        }
        if (columnFamily.equals(DataFileColumnFamily.NAME)) {
            try {
                DataFileValue dfv = new DataFileValue(columnUpdate.getValue());
                if (dfv.getSize() < 0 || dfv.getNumEntries() < 0) {
                    violations = addViolation(violations, 1);
                }
            } catch (NumberFormatException | ArrayIndexOutOfBoundsException nfe) {
                violations = addViolation(violations, 1);
            }
        } else if (columnFamily.equals(ScanFileColumnFamily.NAME)) {
        } else if (columnFamily.equals(BulkFileColumnFamily.NAME)) {
            if (!columnUpdate.isDeleted() && !checkedBulk) {
                // splits, which also write the time reference, are allowed to write this reference even
                // when
                // the transaction is not running because the other half of the tablet is holding a
                // reference
                // to the file.
                boolean isSplitMutation = false;
                // When a tablet is assigned, it re-writes the metadata. It should probably only update
                // the location information,
                // but it writes everything. We allow it to re-write the bulk information if it is setting
                // the location.
                // See ACCUMULO-1230.
                boolean isLocationMutation = false;
                HashSet<Text> dataFiles = new HashSet<>();
                HashSet<Text> loadedFiles = new HashSet<>();
                String tidString = new String(columnUpdate.getValue(), UTF_8);
                int otherTidCount = 0;
                for (ColumnUpdate update : mutation.getUpdates()) {
                    if (new ColumnFQ(update).equals(ServerColumnFamily.DIRECTORY_COLUMN)) {
                        isSplitMutation = true;
                    } else if (new Text(update.getColumnFamily()).equals(CurrentLocationColumnFamily.NAME)) {
                        isLocationMutation = true;
                    } else if (new Text(update.getColumnFamily()).equals(DataFileColumnFamily.NAME)) {
                        dataFiles.add(new Text(update.getColumnQualifier()));
                    } else if (new Text(update.getColumnFamily()).equals(BulkFileColumnFamily.NAME)) {
                        loadedFiles.add(new Text(update.getColumnQualifier()));
                        if (!new String(update.getValue(), UTF_8).equals(tidString)) {
                            otherTidCount++;
                        }
                    }
                }
                if (!isSplitMutation && !isLocationMutation) {
                    long tid = BulkFileColumnFamily.getBulkLoadTid(new Value(tidString));
                    try {
                        if (otherTidCount > 0 || !dataFiles.equals(loadedFiles) || !getArbitrator(context).transactionAlive(Constants.BULK_ARBITRATOR_TYPE, tid)) {
                            violations = addViolation(violations, 8);
                        }
                    } catch (Exception ex) {
                        violations = addViolation(violations, 8);
                    }
                }
                checkedBulk = true;
            }
        } else {
            if (!isValidColumn(columnUpdate)) {
                violations = addViolation(violations, 2);
            } else if (new ColumnFQ(columnUpdate).equals(TabletColumnFamily.PREV_ROW_COLUMN) && columnUpdate.getValue().length > 0 && (violations == null || !violations.contains((short) 4))) {
                KeyExtent ke = KeyExtent.fromMetaRow(new Text(mutation.getRow()));
                Text per = TabletColumnFamily.decodePrevEndRow(new Value(columnUpdate.getValue()));
                boolean prevEndRowLessThanEndRow = per == null || ke.endRow() == null || per.compareTo(ke.endRow()) < 0;
                if (!prevEndRowLessThanEndRow) {
                    violations = addViolation(violations, 3);
                }
            } else if (new ColumnFQ(columnUpdate).equals(ServerColumnFamily.LOCK_COLUMN)) {
                if (zooCache == null) {
                    zooCache = new ZooCache(context.getZooReaderWriter(), null);
                    CleanerUtil.zooCacheClearer(this, zooCache);
                }
                if (zooRoot == null) {
                    zooRoot = context.getZooKeeperRoot();
                }
                boolean lockHeld = false;
                String lockId = new String(columnUpdate.getValue(), UTF_8);
                try {
                    lockHeld = ServiceLock.isLockHeld(zooCache, new ZooUtil.LockID(zooRoot, lockId));
                } catch (Exception e) {
                    log.debug("Failed to verify lock was held {} {}", lockId, e.getMessage());
                }
                if (!lockHeld) {
                    violations = addViolation(violations, 7);
                }
            }
        }
    }
    if (violations != null) {
        log.debug("violating metadata mutation : {}", new String(mutation.getRow(), UTF_8));
        for (ColumnUpdate update : mutation.getUpdates()) {
            log.debug(" update: {}:{} value {}", new String(update.getColumnFamily(), UTF_8), new String(update.getColumnQualifier(), UTF_8), (update.isDeleted() ? "[delete]" : new String(update.getValue(), UTF_8)));
        }
    }
    return violations;
}