Examples with ColumnFQ org.apache.accumulo.core.util.ColumnFQ used on opensource projects

Example 1 with ColumnFQ

use of org.apache.accumulo.core.util.ColumnFQ in project accumulo by apache.

the class MetadataConstraints method check.

@Override
public List<Short> check(Environment env, Mutation mutation) {
    ArrayList<Short> violations = null;
    Collection<ColumnUpdate> colUpdates = mutation.getUpdates();
    // check the row, it should contains at least one ; or end with <
    boolean containsSemiC = false;
    byte[] row = mutation.getRow();
    // always allow rows that fall within reserved areas
    if (row.length > 0 && row[0] == '~')
        return null;
    if (row.length > 2 && row[0] == '!' && row[1] == '!' && row[2] == '~')
        return null;
    for (byte b : row) {
        if (b == ';') {
            containsSemiC = true;
        }
        if (b == ';' || b == '<')
            break;
        if (!validTableNameChars[0xff & b]) {
            violations = addIfNotPresent(violations, 4);
        }
    }
    if (!containsSemiC) {
        // see if last row char is <
        if (row.length == 0 || row[row.length - 1] != '<') {
            violations = addIfNotPresent(violations, 4);
        }
    } else {
        if (row.length == 0) {
            violations = addIfNotPresent(violations, 4);
        }
    }
    if (row.length > 0 && row[0] == '!') {
        if (row.length < 3 || row[1] != '0' || (row[2] != '<' && row[2] != ';')) {
            violations = addIfNotPresent(violations, 4);
        }
    }
    // ensure row is not less than Constants.METADATA_TABLE_ID
    if (new Text(row).compareTo(new Text(MetadataTable.ID.getUtf8())) < 0) {
        violations = addViolation(violations, 5);
    }
    boolean checkedBulk = false;
    for (ColumnUpdate columnUpdate : colUpdates) {
        Text columnFamily = new Text(columnUpdate.getColumnFamily());
        if (columnUpdate.isDeleted()) {
            if (!isValidColumn(columnUpdate)) {
                violations = addViolation(violations, 2);
            }
            continue;
        }
        if (columnUpdate.getValue().length == 0 && !columnFamily.equals(ScanFileColumnFamily.NAME)) {
            violations = addViolation(violations, 6);
        }
        if (columnFamily.equals(DataFileColumnFamily.NAME)) {
            try {
                DataFileValue dfv = new DataFileValue(columnUpdate.getValue());
                if (dfv.getSize() < 0 || dfv.getNumEntries() < 0) {
                    violations = addViolation(violations, 1);
                }
            } catch (NumberFormatException | ArrayIndexOutOfBoundsException nfe) {
                violations = addViolation(violations, 1);
            }
        } else if (columnFamily.equals(ScanFileColumnFamily.NAME)) {
        } else if (columnFamily.equals(TabletsSection.BulkFileColumnFamily.NAME)) {
            if (!columnUpdate.isDeleted() && !checkedBulk) {
                // splits, which also write the time reference, are allowed to write this reference even when
                // the transaction is not running because the other half of the tablet is holding a reference
                // to the file.
                boolean isSplitMutation = false;
                // When a tablet is assigned, it re-writes the metadata. It should probably only update the location information,
                // but it writes everything. We allow it to re-write the bulk information if it is setting the location.
                // See ACCUMULO-1230.
                boolean isLocationMutation = false;
                HashSet<Text> dataFiles = new HashSet<>();
                HashSet<Text> loadedFiles = new HashSet<>();
                String tidString = new String(columnUpdate.getValue(), UTF_8);
                int otherTidCount = 0;
                for (ColumnUpdate update : mutation.getUpdates()) {
                    if (new ColumnFQ(update).equals(TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN)) {
                        isSplitMutation = true;
                    } else if (new Text(update.getColumnFamily()).equals(TabletsSection.CurrentLocationColumnFamily.NAME)) {
                        isLocationMutation = true;
                    } else if (new Text(update.getColumnFamily()).equals(DataFileColumnFamily.NAME)) {
                        dataFiles.add(new Text(update.getColumnQualifier()));
                    } else if (new Text(update.getColumnFamily()).equals(TabletsSection.BulkFileColumnFamily.NAME)) {
                        loadedFiles.add(new Text(update.getColumnQualifier()));
                        if (!new String(update.getValue(), UTF_8).equals(tidString)) {
                            otherTidCount++;
                        }
                    }
                }
                if (!isSplitMutation && !isLocationMutation) {
                    long tid = Long.parseLong(tidString);
                    try {
                        if (otherTidCount > 0 || !dataFiles.equals(loadedFiles) || !getArbitrator().transactionAlive(Constants.BULK_ARBITRATOR_TYPE, tid)) {
                            violations = addViolation(violations, 8);
                        }
                    } catch (Exception ex) {
                        violations = addViolation(violations, 8);
                    }
                }
                checkedBulk = true;
            }
        } else {
            if (!isValidColumn(columnUpdate)) {
                violations = addViolation(violations, 2);
            } else if (new ColumnFQ(columnUpdate).equals(TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN) && columnUpdate.getValue().length > 0 && (violations == null || !violations.contains((short) 4))) {
                KeyExtent ke = new KeyExtent(new Text(mutation.getRow()), (Text) null);
                Text per = KeyExtent.decodePrevEndRow(new Value(columnUpdate.getValue()));
                boolean prevEndRowLessThanEndRow = per == null || ke.getEndRow() == null || per.compareTo(ke.getEndRow()) < 0;
                if (!prevEndRowLessThanEndRow) {
                    violations = addViolation(violations, 3);
                }
            } else if (new ColumnFQ(columnUpdate).equals(TabletsSection.ServerColumnFamily.LOCK_COLUMN)) {
                if (zooCache == null) {
                    zooCache = new ZooCache();
                }
                if (zooRoot == null) {
                    zooRoot = ZooUtil.getRoot(HdfsZooInstance.getInstance());
                }
                boolean lockHeld = false;
                String lockId = new String(columnUpdate.getValue(), UTF_8);
                try {
                    lockHeld = ZooLock.isLockHeld(zooCache, new ZooUtil.LockID(zooRoot, lockId));
                } catch (Exception e) {
                    log.debug("Failed to verify lock was held {} {}", lockId, e.getMessage());
                }
                if (!lockHeld) {
                    violations = addViolation(violations, 7);
                }
            }
        }
    }
    if (violations != null) {
        log.debug("violating metadata mutation : {}", new String(mutation.getRow(), UTF_8));
        for (ColumnUpdate update : mutation.getUpdates()) {
            log.debug(" update: {}:{} value {}", new String(update.getColumnFamily(), UTF_8), new String(update.getColumnQualifier(), UTF_8), (update.isDeleted() ? "[delete]" : new String(update.getValue(), UTF_8)));
        }
    }
    return violations;
}

Example 2 with ColumnFQ

use of org.apache.accumulo.core.util.ColumnFQ in project accumulo by apache.

the class MetadataTableUtil method getTabletEntries.

public static SortedMap<Text, SortedMap<ColumnFQ, Value>> getTabletEntries(SortedMap<Key, Value> tabletKeyValues, List<ColumnFQ> columns) {
    TreeMap<Text, SortedMap<ColumnFQ, Value>> tabletEntries = new TreeMap<>();
    HashSet<ColumnFQ> colSet = null;
    if (columns != null) {
        colSet = new HashSet<>(columns);
    }
    for (Entry<Key, Value> entry : tabletKeyValues.entrySet()) {
        ColumnFQ currentKey = new ColumnFQ(entry.getKey());
        if (columns != null && !colSet.contains(currentKey)) {
            continue;
        }
        Text row = entry.getKey().getRow();
        SortedMap<ColumnFQ, Value> colVals = tabletEntries.get(row);
        if (colVals == null) {
            colVals = new TreeMap<>();
            tabletEntries.put(row, colVals);
        }
        colVals.put(currentKey, entry.getValue());
    }
    return tabletEntries;
}

Example 3 with ColumnFQ

use of org.apache.accumulo.core.util.ColumnFQ in project accumulo by apache.

the class Initialize method createEntriesForTablet.

private static void createEntriesForTablet(TreeMap<Key, Value> map, Tablet tablet) {
    Value EMPTY_SIZE = new DataFileValue(0, 0).encodeAsValue();
    Text extent = new Text(KeyExtent.getMetadataEntry(tablet.tableId, tablet.endRow));
    addEntry(map, extent, DIRECTORY_COLUMN, new Value(tablet.dir.getBytes(UTF_8)));
    addEntry(map, extent, TIME_COLUMN, new Value((TabletTime.LOGICAL_TIME_ID + "0").getBytes(UTF_8)));
    addEntry(map, extent, PREV_ROW_COLUMN, KeyExtent.encodePrevEndRow(tablet.prevEndRow));
    for (String file : tablet.files) {
        addEntry(map, extent, new ColumnFQ(DataFileColumnFamily.NAME, new Text(file)), EMPTY_SIZE);
    }
}

Example 4 with ColumnFQ

use of org.apache.accumulo.core.util.ColumnFQ in project accumulo by apache.

the class TabletServer method verifyTabletInformation.

public static Pair<Text, KeyExtent> verifyTabletInformation(AccumuloServerContext context, KeyExtent extent, TServerInstance instance, final SortedMap<Key, Value> tabletsKeyValues, String clientAddress, ZooLock lock) throws AccumuloSecurityException, DistributedStoreException, AccumuloException {
    Objects.requireNonNull(tabletsKeyValues);
    log.debug("verifying extent {}", extent);
    if (extent.isRootTablet()) {
        return verifyRootTablet(extent, instance);
    }
    Table.ID tableToVerify = MetadataTable.ID;
    if (extent.isMeta())
        tableToVerify = RootTable.ID;
    List<ColumnFQ> columnsToFetch = Arrays.asList(new ColumnFQ[] { TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN, TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN, TabletsSection.TabletColumnFamily.SPLIT_RATIO_COLUMN, TabletsSection.TabletColumnFamily.OLD_PREV_ROW_COLUMN, TabletsSection.ServerColumnFamily.TIME_COLUMN });
    TreeMap<Key, Value> tkv = new TreeMap<>();
    try (ScannerImpl scanner = new ScannerImpl(context, tableToVerify, Authorizations.EMPTY)) {
        scanner.setRange(extent.toMetadataRange());
        for (Entry<Key, Value> entry : scanner) tkv.put(entry.getKey(), entry.getValue());
    }
    // only populate map after success
    tabletsKeyValues.clear();
    tabletsKeyValues.putAll(tkv);
    Text metadataEntry = extent.getMetadataEntry();
    Value dir = checkTabletMetadata(extent, instance, tabletsKeyValues, metadataEntry);
    if (dir == null)
        return null;
    Value oldPrevEndRow = null;
    for (Entry<Key, Value> entry : tabletsKeyValues.entrySet()) {
        if (TabletsSection.TabletColumnFamily.OLD_PREV_ROW_COLUMN.hasColumns(entry.getKey())) {
            oldPrevEndRow = entry.getValue();
        }
    }
    if (oldPrevEndRow != null) {
        SortedMap<Text, SortedMap<ColumnFQ, Value>> tabletEntries;
        tabletEntries = MetadataTableUtil.getTabletEntries(tabletsKeyValues, columnsToFetch);
        KeyExtent fke;
        try {
            fke = MasterMetadataUtil.fixSplit(context, metadataEntry, tabletEntries.get(metadataEntry), instance, lock);
        } catch (IOException e) {
            log.error("Error fixing split {}", metadataEntry);
            throw new AccumuloException(e.toString());
        }
        if (!fke.equals(extent)) {
            return new Pair<>(null, fke);
        }
        // reread and reverify metadata entries now that metadata entries were fixed
        tabletsKeyValues.clear();
        return verifyTabletInformation(context, fke, instance, tabletsKeyValues, clientAddress, lock);
    }
    return new Pair<>(new Text(dir.get()), null);
}

Example 5 with ColumnFQ

use of org.apache.accumulo.core.util.ColumnFQ in project accumulo by apache.

the class SplitRecoveryIT method ensureTabletHasNoUnexpectedMetadataEntries.

private void ensureTabletHasNoUnexpectedMetadataEntries(AccumuloServerContext context, KeyExtent extent, SortedMap<FileRef, DataFileValue> expectedMapFiles) throws Exception {
    try (Scanner scanner = new ScannerImpl(context, MetadataTable.ID, Authorizations.EMPTY)) {
        scanner.setRange(extent.toMetadataRange());
        HashSet<ColumnFQ> expectedColumns = new HashSet<>();
        expectedColumns.add(TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN);
        expectedColumns.add(TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN);
        expectedColumns.add(TabletsSection.ServerColumnFamily.TIME_COLUMN);
        expectedColumns.add(TabletsSection.ServerColumnFamily.LOCK_COLUMN);
        HashSet<Text> expectedColumnFamilies = new HashSet<>();
        expectedColumnFamilies.add(DataFileColumnFamily.NAME);
        expectedColumnFamilies.add(TabletsSection.FutureLocationColumnFamily.NAME);
        expectedColumnFamilies.add(TabletsSection.CurrentLocationColumnFamily.NAME);
        expectedColumnFamilies.add(TabletsSection.LastLocationColumnFamily.NAME);
        expectedColumnFamilies.add(TabletsSection.BulkFileColumnFamily.NAME);
        Iterator<Entry<Key, Value>> iter = scanner.iterator();
        while (iter.hasNext()) {
            Key key = iter.next().getKey();
            if (!key.getRow().equals(extent.getMetadataEntry())) {
                throw new Exception("Tablet " + extent + " contained unexpected " + MetadataTable.NAME + " entry " + key);
            }
            if (expectedColumnFamilies.contains(key.getColumnFamily())) {
                continue;
            }
            if (expectedColumns.remove(new ColumnFQ(key))) {
                continue;
            }
            throw new Exception("Tablet " + extent + " contained unexpected " + MetadataTable.NAME + " entry " + key);
        }
        System.out.println("expectedColumns " + expectedColumns);
        if (expectedColumns.size() > 1 || (expectedColumns.size() == 1)) {
            throw new Exception("Not all expected columns seen " + extent + " " + expectedColumns);
        }
        SortedMap<FileRef, DataFileValue> fixedMapFiles = MetadataTableUtil.getDataFileSizes(extent, context);
        verifySame(expectedMapFiles, fixedMapFiles);
    }
}