Examples with KeyExtent org.apache.accumulo.core.dataImpl.KeyExtent used on opensource projects

Example 41 with KeyExtent

use of org.apache.accumulo.core.dataImpl.KeyExtent in project accumulo by apache.

the class TabletGroupWatcher method mergeMetadataRecords.

private void mergeMetadataRecords(MergeInfo info) throws AccumuloException {
    KeyExtent range = info.getExtent();
    Manager.log.debug("Merging metadata for {}", range);
    KeyExtent stop = getHighTablet(range);
    Manager.log.debug("Highest tablet is {}", stop);
    Value firstPrevRowValue = null;
    Text stopRow = stop.toMetaRow();
    Text start = range.prevEndRow();
    if (start == null) {
        start = new Text();
    }
    Range scanRange = new Range(TabletsSection.encodeRow(range.tableId(), start), false, stopRow, false);
    String targetSystemTable = MetadataTable.NAME;
    if (range.isMeta()) {
        targetSystemTable = RootTable.NAME;
    }
    AccumuloClient client = manager.getContext();
    try (BatchWriter bw = client.createBatchWriter(targetSystemTable)) {
        long fileCount = 0;
        // Make file entries in highest tablet
        Scanner scanner = client.createScanner(targetSystemTable, Authorizations.EMPTY);
        scanner.setRange(scanRange);
        TabletColumnFamily.PREV_ROW_COLUMN.fetch(scanner);
        ServerColumnFamily.TIME_COLUMN.fetch(scanner);
        ServerColumnFamily.DIRECTORY_COLUMN.fetch(scanner);
        scanner.fetchColumnFamily(DataFileColumnFamily.NAME);
        Mutation m = new Mutation(stopRow);
        MetadataTime maxLogicalTime = null;
        for (Entry<Key, Value> entry : scanner) {
            Key key = entry.getKey();
            Value value = entry.getValue();
            if (key.getColumnFamily().equals(DataFileColumnFamily.NAME)) {
                m.put(key.getColumnFamily(), key.getColumnQualifier(), value);
                fileCount++;
            } else if (TabletColumnFamily.PREV_ROW_COLUMN.hasColumns(key) && firstPrevRowValue == null) {
                Manager.log.debug("prevRow entry for lowest tablet is {}", value);
                firstPrevRowValue = new Value(value);
            } else if (ServerColumnFamily.TIME_COLUMN.hasColumns(key)) {
                maxLogicalTime = TabletTime.maxMetadataTime(maxLogicalTime, MetadataTime.parse(value.toString()));
            } else if (ServerColumnFamily.DIRECTORY_COLUMN.hasColumns(key)) {
                String uri = GcVolumeUtil.getDeleteTabletOnAllVolumesUri(range.tableId(), value.toString());
                bw.addMutation(manager.getContext().getAmple().createDeleteMutation(uri));
            }
        }
        // read the logical time from the last tablet in the merge range, it is not included in
        // the loop above
        scanner = client.createScanner(targetSystemTable, Authorizations.EMPTY);
        scanner.setRange(new Range(stopRow));
        ServerColumnFamily.TIME_COLUMN.fetch(scanner);
        scanner.fetchColumnFamily(ExternalCompactionColumnFamily.NAME);
        Set<String> extCompIds = new HashSet<>();
        for (Entry<Key, Value> entry : scanner) {
            if (ServerColumnFamily.TIME_COLUMN.hasColumns(entry.getKey())) {
                maxLogicalTime = TabletTime.maxMetadataTime(maxLogicalTime, MetadataTime.parse(entry.getValue().toString()));
            } else if (ExternalCompactionColumnFamily.NAME.equals(entry.getKey().getColumnFamily())) {
                extCompIds.add(entry.getKey().getColumnQualifierData().toString());
            }
        }
        if (maxLogicalTime != null)
            ServerColumnFamily.TIME_COLUMN.put(m, new Value(maxLogicalTime.encode()));
        // delete any entries for external compactions
        extCompIds.stream().forEach(ecid -> m.putDelete(ExternalCompactionColumnFamily.STR_NAME, ecid));
        if (!m.getUpdates().isEmpty()) {
            bw.addMutation(m);
        }
        bw.flush();
        Manager.log.debug("Moved {} files to {}", fileCount, stop);
        if (firstPrevRowValue == null) {
            Manager.log.debug("tablet already merged");
            return;
        }
        stop = new KeyExtent(stop.tableId(), stop.endRow(), TabletColumnFamily.decodePrevEndRow(firstPrevRowValue));
        Mutation updatePrevRow = TabletColumnFamily.createPrevRowMutation(stop);
        Manager.log.debug("Setting the prevRow for last tablet: {}", stop);
        bw.addMutation(updatePrevRow);
        bw.flush();
        deleteTablets(info, scanRange, bw, client);
        // Clean-up the last chopped marker
        var m2 = new Mutation(stopRow);
        ChoppedColumnFamily.CHOPPED_COLUMN.putDelete(m2);
        bw.addMutation(m2);
        bw.flush();
    } catch (Exception ex) {
        throw new AccumuloException(ex);
    }
}

Example 42 with KeyExtent

use of org.apache.accumulo.core.dataImpl.KeyExtent in project accumulo by apache.

the class TabletGroupWatcher method deleteTablets.

private void deleteTablets(MergeInfo info) throws AccumuloException {
    KeyExtent extent = info.getExtent();
    String targetSystemTable = extent.isMeta() ? RootTable.NAME : MetadataTable.NAME;
    Manager.log.debug("Deleting tablets for {}", extent);
    MetadataTime metadataTime = null;
    KeyExtent followingTablet = null;
    if (extent.endRow() != null) {
        Key nextExtent = new Key(extent.endRow()).followingKey(PartialKey.ROW);
        followingTablet = getHighTablet(new KeyExtent(extent.tableId(), nextExtent.getRow(), extent.endRow()));
        Manager.log.debug("Found following tablet {}", followingTablet);
    }
    try {
        AccumuloClient client = manager.getContext();
        ServerContext context = manager.getContext();
        Ample ample = context.getAmple();
        Text start = extent.prevEndRow();
        if (start == null) {
            start = new Text();
        }
        Manager.log.debug("Making file deletion entries for {}", extent);
        Range deleteRange = new Range(TabletsSection.encodeRow(extent.tableId(), start), false, TabletsSection.encodeRow(extent.tableId(), extent.endRow()), true);
        Scanner scanner = client.createScanner(targetSystemTable, Authorizations.EMPTY);
        scanner.setRange(deleteRange);
        ServerColumnFamily.DIRECTORY_COLUMN.fetch(scanner);
        ServerColumnFamily.TIME_COLUMN.fetch(scanner);
        scanner.fetchColumnFamily(DataFileColumnFamily.NAME);
        scanner.fetchColumnFamily(CurrentLocationColumnFamily.NAME);
        Set<String> datafiles = new TreeSet<>();
        for (Entry<Key, Value> entry : scanner) {
            Key key = entry.getKey();
            if (key.compareColumnFamily(DataFileColumnFamily.NAME) == 0) {
                datafiles.add(TabletFileUtil.validate(key.getColumnQualifierData().toString()));
                if (datafiles.size() > 1000) {
                    ample.putGcFileAndDirCandidates(extent.tableId(), datafiles);
                    datafiles.clear();
                }
            } else if (ServerColumnFamily.TIME_COLUMN.hasColumns(key)) {
                metadataTime = MetadataTime.parse(entry.getValue().toString());
            } else if (key.compareColumnFamily(CurrentLocationColumnFamily.NAME) == 0) {
                throw new IllegalStateException("Tablet " + key.getRow() + " is assigned during a merge!");
            } else if (ServerColumnFamily.DIRECTORY_COLUMN.hasColumns(key)) {
                String path = GcVolumeUtil.getDeleteTabletOnAllVolumesUri(extent.tableId(), entry.getValue().toString());
                datafiles.add(path);
                if (datafiles.size() > 1000) {
                    ample.putGcFileAndDirCandidates(extent.tableId(), datafiles);
                    datafiles.clear();
                }
            }
        }
        ample.putGcFileAndDirCandidates(extent.tableId(), datafiles);
        BatchWriter bw = client.createBatchWriter(targetSystemTable);
        try {
            deleteTablets(info, deleteRange, bw, client);
        } finally {
            bw.close();
        }
        if (followingTablet != null) {
            Manager.log.debug("Updating prevRow of {} to {}", followingTablet, extent.prevEndRow());
            bw = client.createBatchWriter(targetSystemTable);
            try {
                Mutation m = new Mutation(followingTablet.toMetaRow());
                TabletColumnFamily.PREV_ROW_COLUMN.put(m, TabletColumnFamily.encodePrevEndRow(extent.prevEndRow()));
                ChoppedColumnFamily.CHOPPED_COLUMN.putDelete(m);
                bw.addMutation(m);
                bw.flush();
            } finally {
                bw.close();
            }
        } else {
            // Recreate the default tablet to hold the end of the table
            MetadataTableUtil.addTablet(new KeyExtent(extent.tableId(), null, extent.prevEndRow()), ServerColumnFamily.DEFAULT_TABLET_DIR_NAME, manager.getContext(), metadataTime.getType(), manager.managerLock);
        }
    } catch (RuntimeException | TableNotFoundException ex) {
        throw new AccumuloException(ex);
    }
}

Example 43 with KeyExtent

use of org.apache.accumulo.core.dataImpl.KeyExtent in project accumulo by apache.

the class TabletGroupWatcher method repairMetadata.

private void repairMetadata(Text row) {
    Manager.log.debug("Attempting repair on {}", row);
    // Attempt to find the dead server entry and remove it.
    try {
        Map<Key, Value> future = new HashMap<>();
        Map<Key, Value> assigned = new HashMap<>();
        KeyExtent extent = KeyExtent.fromMetaRow(row);
        String table = MetadataTable.NAME;
        if (extent.isMeta())
            table = RootTable.NAME;
        Scanner scanner = manager.getContext().createScanner(table, Authorizations.EMPTY);
        scanner.fetchColumnFamily(CurrentLocationColumnFamily.NAME);
        scanner.fetchColumnFamily(FutureLocationColumnFamily.NAME);
        scanner.setRange(new Range(row));
        for (Entry<Key, Value> entry : scanner) {
            if (entry.getKey().getColumnFamily().equals(CurrentLocationColumnFamily.NAME)) {
                assigned.put(entry.getKey(), entry.getValue());
            } else if (entry.getKey().getColumnFamily().equals(FutureLocationColumnFamily.NAME)) {
                future.put(entry.getKey(), entry.getValue());
            }
        }
        if (!future.isEmpty() && !assigned.isEmpty()) {
            Manager.log.warn("Found a tablet assigned and hosted, attempting to repair");
        } else if (future.size() > 1 && assigned.isEmpty()) {
            Manager.log.warn("Found a tablet assigned to multiple servers, attempting to repair");
        } else if (future.isEmpty() && assigned.size() > 1) {
            Manager.log.warn("Found a tablet hosted on multiple servers, attempting to repair");
        } else {
            Manager.log.info("Attempted a repair, but nothing seems to be obviously wrong. {} {}", assigned, future);
            return;
        }
        Iterator<Entry<Key, Value>> iter = Iterators.concat(future.entrySet().iterator(), assigned.entrySet().iterator());
        while (iter.hasNext()) {
            Entry<Key, Value> entry = iter.next();
            TServerInstance alive = manager.tserverSet.find(entry.getValue().toString());
            if (alive == null) {
                Manager.log.info("Removing entry  {}", entry);
                BatchWriter bw = manager.getContext().createBatchWriter(table);
                Mutation m = new Mutation(entry.getKey().getRow());
                m.putDelete(entry.getKey().getColumnFamily(), entry.getKey().getColumnQualifier());
                bw.addMutation(m);
                bw.close();
                return;
            }
        }
        Manager.log.error("Metadata table is inconsistent at {} and all assigned/future tservers are still online.", row);
    } catch (Exception e) {
        Manager.log.error("Error attempting repair of metadata " + row + ": " + e, e);
    }
}

Example 44 with KeyExtent

use of org.apache.accumulo.core.dataImpl.KeyExtent in project accumulo by apache.

the class CopyFailed method call.

@Override
public Repo<Manager> call(long tid, Manager manager) throws Exception {
    // This needs to execute after the arbiter is stopped
    manager.updateBulkImportStatus(source, BulkImportState.COPY_FILES);
    VolumeManager fs = manager.getVolumeManager();
    if (!fs.exists(new Path(error, BulkImport.FAILURES_TXT)))
        return new CleanUpBulkImport(tableId, source, bulk, error);
    var failures = new HashSet<Path>();
    var loadedFailures = new HashSet<Path>();
    try (BufferedReader in = new BufferedReader(new InputStreamReader(fs.open(new Path(error, BulkImport.FAILURES_TXT)), UTF_8))) {
        String line = null;
        while ((line = in.readLine()) != null) {
            Path path = new Path(line);
            if (!fs.exists(new Path(error, path.getName())))
                failures.add(path);
        }
    }
    /*
     * I thought I could move files that have no file references in the table. However its possible
     * a clone references a file. Therefore only move files that have no loaded markers.
     */
    // determine which failed files were loaded
    AccumuloClient client = manager.getContext();
    try (Scanner mscanner = new IsolatedScanner(client.createScanner(MetadataTable.NAME, Authorizations.EMPTY))) {
        mscanner.setRange(new KeyExtent(tableId, null, null).toMetaRange());
        mscanner.fetchColumnFamily(BulkFileColumnFamily.NAME);
        for (Entry<Key, Value> entry : mscanner) {
            if (BulkFileColumnFamily.getBulkLoadTid(entry.getValue()) == tid) {
                Path loadedFile = new Path(TabletFileUtil.validate(entry.getKey().getColumnQualifierData().toString()));
                if (failures.remove(loadedFile)) {
                    loadedFailures.add(loadedFile);
                }
            }
        }
    }
    // move failed files that were not loaded
    for (Path orig : failures) {
        Path dest = new Path(error, orig.getName());
        fs.rename(orig, dest);
        log.debug(FateTxId.formatTid(tid) + " renamed " + orig + " to " + dest + ": import failed");
    }
    if (!loadedFailures.isEmpty()) {
        DistributedWorkQueue bifCopyQueue = new DistributedWorkQueue(Constants.ZROOT + "/" + manager.getInstanceID() + Constants.ZBULK_FAILED_COPYQ, manager.getConfiguration(), manager.getContext());
        HashSet<String> workIds = new HashSet<>();
        for (Path orig : loadedFailures) {
            Path dest = new Path(error, orig.getName());
            if (fs.exists(dest))
                continue;
            bifCopyQueue.addWork(orig.getName(), (orig + "," + dest).getBytes(UTF_8));
            workIds.add(orig.getName());
            log.debug(FateTxId.formatTid(tid) + " added to copyq: " + orig + " to " + dest + ": failed");
        }
        bifCopyQueue.waitUntilDone(workIds);
    }
    fs.deleteRecursively(new Path(error, BulkImport.FAILURES_TXT));
    return new CleanUpBulkImport(tableId, source, bulk, error);
}

Example 45 with KeyExtent

use of org.apache.accumulo.core.dataImpl.KeyExtent in project accumulo by apache.

the class PrepBulkImport method sanityCheckLoadMapping.

/**
 * Checks a load mapping to ensure all of the rows in the mapping exists in the table and that no
 * file goes to too many tablets.
 */
@VisibleForTesting
static void sanityCheckLoadMapping(String tableId, LoadMappingIterator lmi, TabletIterFactory tabletIterFactory, int maxNumTablets, long tid) throws Exception {
    var currRange = lmi.next();
    Text startRow = currRange.getKey().prevEndRow();
    Iterator<KeyExtent> tabletIter = tabletIterFactory.newTabletIter(startRow);
    KeyExtent currTablet = tabletIter.next();
    var fileCounts = new HashMap<String, Integer>();
    int count;
    if (!tabletIter.hasNext() && equals(KeyExtent::prevEndRow, currTablet, currRange.getKey()) && equals(KeyExtent::endRow, currTablet, currRange.getKey()))
        currRange = null;
    while (tabletIter.hasNext()) {
        if (currRange == null) {
            if (!lmi.hasNext()) {
                break;
            }
            currRange = lmi.next();
        }
        while (!equals(KeyExtent::prevEndRow, currTablet, currRange.getKey()) && tabletIter.hasNext()) {
            currTablet = tabletIter.next();
        }
        boolean matchedPrevRow = equals(KeyExtent::prevEndRow, currTablet, currRange.getKey());
        count = matchedPrevRow ? 1 : 0;
        while (!equals(KeyExtent::endRow, currTablet, currRange.getKey()) && tabletIter.hasNext()) {
            currTablet = tabletIter.next();
            count++;
        }
        if (!matchedPrevRow || !equals(KeyExtent::endRow, currTablet, currRange.getKey())) {
            break;
        }
        if (maxNumTablets > 0) {
            int fc = count;
            currRange.getValue().forEach(fileInfo -> fileCounts.merge(fileInfo.getFileName(), fc, Integer::sum));
        }
        currRange = null;
    }
    if (currRange != null || lmi.hasNext()) {
        // merge happened after the mapping was generated and before the table lock was acquired
        throw new AcceptableThriftTableOperationException(tableId, null, TableOperation.BULK_IMPORT, TableOperationExceptionType.BULK_CONCURRENT_MERGE, "Concurrent merge happened");
    }
    if (maxNumTablets > 0) {
        fileCounts.values().removeIf(c -> c <= maxNumTablets);
        if (!fileCounts.isEmpty()) {
            throw new AcceptableThriftTableOperationException(tableId, null, TableOperation.BULK_IMPORT, TableOperationExceptionType.OTHER, "Files overlap the configured max (" + maxNumTablets + ") number of tablets: " + new TreeMap<>(fileCounts));
        }
    }
}