Examples with AccumuloClient org.apache.accumulo.core.client.AccumuloClient used on opensource projects

Example 1 with AccumuloClient

use of org.apache.accumulo.core.client.AccumuloClient in project accumulo by apache.

the class TabletGroupWatcher method getHighTablet.

private KeyExtent getHighTablet(KeyExtent range) throws AccumuloException {
    try {
        AccumuloClient client = manager.getContext();
        Scanner scanner = client.createScanner(range.isMeta() ? RootTable.NAME : MetadataTable.NAME, Authorizations.EMPTY);
        TabletColumnFamily.PREV_ROW_COLUMN.fetch(scanner);
        KeyExtent start = new KeyExtent(range.tableId(), range.endRow(), null);
        scanner.setRange(new Range(start.toMetaRow(), null));
        Iterator<Entry<Key, Value>> iterator = scanner.iterator();
        if (!iterator.hasNext()) {
            throw new AccumuloException("No last tablet for a merge " + range);
        }
        Entry<Key, Value> entry = iterator.next();
        KeyExtent highTablet = KeyExtent.fromMetaPrevRow(entry);
        if (!highTablet.tableId().equals(range.tableId())) {
            throw new AccumuloException("No last tablet for merge " + range + " " + highTablet);
        }
        return highTablet;
    } catch (Exception ex) {
        throw new AccumuloException("Unexpected failure finding the last tablet for a merge " + range, ex);
    }
}

Example 2 with AccumuloClient

use of org.apache.accumulo.core.client.AccumuloClient in project accumulo by apache.

the class TabletGroupWatcher method mergeMetadataRecords.

private void mergeMetadataRecords(MergeInfo info) throws AccumuloException {
    KeyExtent range = info.getExtent();
    Manager.log.debug("Merging metadata for {}", range);
    KeyExtent stop = getHighTablet(range);
    Manager.log.debug("Highest tablet is {}", stop);
    Value firstPrevRowValue = null;
    Text stopRow = stop.toMetaRow();
    Text start = range.prevEndRow();
    if (start == null) {
        start = new Text();
    }
    Range scanRange = new Range(TabletsSection.encodeRow(range.tableId(), start), false, stopRow, false);
    String targetSystemTable = MetadataTable.NAME;
    if (range.isMeta()) {
        targetSystemTable = RootTable.NAME;
    }
    AccumuloClient client = manager.getContext();
    try (BatchWriter bw = client.createBatchWriter(targetSystemTable)) {
        long fileCount = 0;
        // Make file entries in highest tablet
        Scanner scanner = client.createScanner(targetSystemTable, Authorizations.EMPTY);
        scanner.setRange(scanRange);
        TabletColumnFamily.PREV_ROW_COLUMN.fetch(scanner);
        ServerColumnFamily.TIME_COLUMN.fetch(scanner);
        ServerColumnFamily.DIRECTORY_COLUMN.fetch(scanner);
        scanner.fetchColumnFamily(DataFileColumnFamily.NAME);
        Mutation m = new Mutation(stopRow);
        MetadataTime maxLogicalTime = null;
        for (Entry<Key, Value> entry : scanner) {
            Key key = entry.getKey();
            Value value = entry.getValue();
            if (key.getColumnFamily().equals(DataFileColumnFamily.NAME)) {
                m.put(key.getColumnFamily(), key.getColumnQualifier(), value);
                fileCount++;
            } else if (TabletColumnFamily.PREV_ROW_COLUMN.hasColumns(key) && firstPrevRowValue == null) {
                Manager.log.debug("prevRow entry for lowest tablet is {}", value);
                firstPrevRowValue = new Value(value);
            } else if (ServerColumnFamily.TIME_COLUMN.hasColumns(key)) {
                maxLogicalTime = TabletTime.maxMetadataTime(maxLogicalTime, MetadataTime.parse(value.toString()));
            } else if (ServerColumnFamily.DIRECTORY_COLUMN.hasColumns(key)) {
                String uri = GcVolumeUtil.getDeleteTabletOnAllVolumesUri(range.tableId(), value.toString());
                bw.addMutation(manager.getContext().getAmple().createDeleteMutation(uri));
            }
        }
        // read the logical time from the last tablet in the merge range, it is not included in
        // the loop above
        scanner = client.createScanner(targetSystemTable, Authorizations.EMPTY);
        scanner.setRange(new Range(stopRow));
        ServerColumnFamily.TIME_COLUMN.fetch(scanner);
        scanner.fetchColumnFamily(ExternalCompactionColumnFamily.NAME);
        Set<String> extCompIds = new HashSet<>();
        for (Entry<Key, Value> entry : scanner) {
            if (ServerColumnFamily.TIME_COLUMN.hasColumns(entry.getKey())) {
                maxLogicalTime = TabletTime.maxMetadataTime(maxLogicalTime, MetadataTime.parse(entry.getValue().toString()));
            } else if (ExternalCompactionColumnFamily.NAME.equals(entry.getKey().getColumnFamily())) {
                extCompIds.add(entry.getKey().getColumnQualifierData().toString());
            }
        }
        if (maxLogicalTime != null)
            ServerColumnFamily.TIME_COLUMN.put(m, new Value(maxLogicalTime.encode()));
        // delete any entries for external compactions
        extCompIds.stream().forEach(ecid -> m.putDelete(ExternalCompactionColumnFamily.STR_NAME, ecid));
        if (!m.getUpdates().isEmpty()) {
            bw.addMutation(m);
        }
        bw.flush();
        Manager.log.debug("Moved {} files to {}", fileCount, stop);
        if (firstPrevRowValue == null) {
            Manager.log.debug("tablet already merged");
            return;
        }
        stop = new KeyExtent(stop.tableId(), stop.endRow(), TabletColumnFamily.decodePrevEndRow(firstPrevRowValue));
        Mutation updatePrevRow = TabletColumnFamily.createPrevRowMutation(stop);
        Manager.log.debug("Setting the prevRow for last tablet: {}", stop);
        bw.addMutation(updatePrevRow);
        bw.flush();
        deleteTablets(info, scanRange, bw, client);
        // Clean-up the last chopped marker
        var m2 = new Mutation(stopRow);
        ChoppedColumnFamily.CHOPPED_COLUMN.putDelete(m2);
        bw.addMutation(m2);
        bw.flush();
    } catch (Exception ex) {
        throw new AccumuloException(ex);
    }
}

Example 3 with AccumuloClient

use of org.apache.accumulo.core.client.AccumuloClient in project accumulo by apache.

the class TabletGroupWatcher method deleteTablets.

private void deleteTablets(MergeInfo info) throws AccumuloException {
    KeyExtent extent = info.getExtent();
    String targetSystemTable = extent.isMeta() ? RootTable.NAME : MetadataTable.NAME;
    Manager.log.debug("Deleting tablets for {}", extent);
    MetadataTime metadataTime = null;
    KeyExtent followingTablet = null;
    if (extent.endRow() != null) {
        Key nextExtent = new Key(extent.endRow()).followingKey(PartialKey.ROW);
        followingTablet = getHighTablet(new KeyExtent(extent.tableId(), nextExtent.getRow(), extent.endRow()));
        Manager.log.debug("Found following tablet {}", followingTablet);
    }
    try {
        AccumuloClient client = manager.getContext();
        ServerContext context = manager.getContext();
        Ample ample = context.getAmple();
        Text start = extent.prevEndRow();
        if (start == null) {
            start = new Text();
        }
        Manager.log.debug("Making file deletion entries for {}", extent);
        Range deleteRange = new Range(TabletsSection.encodeRow(extent.tableId(), start), false, TabletsSection.encodeRow(extent.tableId(), extent.endRow()), true);
        Scanner scanner = client.createScanner(targetSystemTable, Authorizations.EMPTY);
        scanner.setRange(deleteRange);
        ServerColumnFamily.DIRECTORY_COLUMN.fetch(scanner);
        ServerColumnFamily.TIME_COLUMN.fetch(scanner);
        scanner.fetchColumnFamily(DataFileColumnFamily.NAME);
        scanner.fetchColumnFamily(CurrentLocationColumnFamily.NAME);
        Set<String> datafiles = new TreeSet<>();
        for (Entry<Key, Value> entry : scanner) {
            Key key = entry.getKey();
            if (key.compareColumnFamily(DataFileColumnFamily.NAME) == 0) {
                datafiles.add(TabletFileUtil.validate(key.getColumnQualifierData().toString()));
                if (datafiles.size() > 1000) {
                    ample.putGcFileAndDirCandidates(extent.tableId(), datafiles);
                    datafiles.clear();
                }
            } else if (ServerColumnFamily.TIME_COLUMN.hasColumns(key)) {
                metadataTime = MetadataTime.parse(entry.getValue().toString());
            } else if (key.compareColumnFamily(CurrentLocationColumnFamily.NAME) == 0) {
                throw new IllegalStateException("Tablet " + key.getRow() + " is assigned during a merge!");
            } else if (ServerColumnFamily.DIRECTORY_COLUMN.hasColumns(key)) {
                String path = GcVolumeUtil.getDeleteTabletOnAllVolumesUri(extent.tableId(), entry.getValue().toString());
                datafiles.add(path);
                if (datafiles.size() > 1000) {
                    ample.putGcFileAndDirCandidates(extent.tableId(), datafiles);
                    datafiles.clear();
                }
            }
        }
        ample.putGcFileAndDirCandidates(extent.tableId(), datafiles);
        BatchWriter bw = client.createBatchWriter(targetSystemTable);
        try {
            deleteTablets(info, deleteRange, bw, client);
        } finally {
            bw.close();
        }
        if (followingTablet != null) {
            Manager.log.debug("Updating prevRow of {} to {}", followingTablet, extent.prevEndRow());
            bw = client.createBatchWriter(targetSystemTable);
            try {
                Mutation m = new Mutation(followingTablet.toMetaRow());
                TabletColumnFamily.PREV_ROW_COLUMN.put(m, TabletColumnFamily.encodePrevEndRow(extent.prevEndRow()));
                ChoppedColumnFamily.CHOPPED_COLUMN.putDelete(m);
                bw.addMutation(m);
                bw.flush();
            } finally {
                bw.close();
            }
        } else {
            // Recreate the default tablet to hold the end of the table
            MetadataTableUtil.addTablet(new KeyExtent(extent.tableId(), null, extent.prevEndRow()), ServerColumnFamily.DEFAULT_TABLET_DIR_NAME, manager.getContext(), metadataTime.getType(), manager.managerLock);
        }
    } catch (RuntimeException | TableNotFoundException ex) {
        throw new AccumuloException(ex);
    }
}

Example 4 with AccumuloClient

use of org.apache.accumulo.core.client.AccumuloClient in project accumulo by apache.

the class CleanUpBulkImport method call.

@Override
public Repo<Manager> call(long tid, Manager manager) throws Exception {
    manager.updateBulkImportStatus(source, BulkImportState.CLEANUP);
    log.debug("removing the bulkDir processing flag file in " + bulk);
    Path bulkDir = new Path(bulk);
    MetadataTableUtil.removeBulkLoadInProgressFlag(manager.getContext(), "/" + bulkDir.getParent().getName() + "/" + bulkDir.getName());
    manager.getContext().getAmple().putGcFileAndDirCandidates(tableId, Collections.singleton(bulkDir.toString()));
    log.debug("removing the metadata table markers for loaded files");
    AccumuloClient client = manager.getContext();
    MetadataTableUtil.removeBulkLoadEntries(client, tableId, tid);
    log.debug("releasing HDFS reservations for " + source + " and " + error);
    Utils.unreserveHdfsDirectory(manager, source, tid);
    Utils.unreserveHdfsDirectory(manager, error, tid);
    Utils.getReadLock(manager, tableId, tid).unlock();
    log.debug("completing bulkDir import transaction " + FateTxId.formatTid(tid));
    ZooArbitrator.cleanup(manager.getContext(), Constants.BULK_ARBITRATOR_TYPE, tid);
    manager.removeBulkImportStatus(source);
    return null;
}

Example 5 with AccumuloClient

use of org.apache.accumulo.core.client.AccumuloClient in project accumulo by apache.

the class CopyFailed method call.

@Override
public Repo<Manager> call(long tid, Manager manager) throws Exception {
    // This needs to execute after the arbiter is stopped
    manager.updateBulkImportStatus(source, BulkImportState.COPY_FILES);
    VolumeManager fs = manager.getVolumeManager();
    if (!fs.exists(new Path(error, BulkImport.FAILURES_TXT)))
        return new CleanUpBulkImport(tableId, source, bulk, error);
    var failures = new HashSet<Path>();
    var loadedFailures = new HashSet<Path>();
    try (BufferedReader in = new BufferedReader(new InputStreamReader(fs.open(new Path(error, BulkImport.FAILURES_TXT)), UTF_8))) {
        String line = null;
        while ((line = in.readLine()) != null) {
            Path path = new Path(line);
            if (!fs.exists(new Path(error, path.getName())))
                failures.add(path);
        }
    }
    /*
     * I thought I could move files that have no file references in the table. However its possible
     * a clone references a file. Therefore only move files that have no loaded markers.
     */
    // determine which failed files were loaded
    AccumuloClient client = manager.getContext();
    try (Scanner mscanner = new IsolatedScanner(client.createScanner(MetadataTable.NAME, Authorizations.EMPTY))) {
        mscanner.setRange(new KeyExtent(tableId, null, null).toMetaRange());
        mscanner.fetchColumnFamily(BulkFileColumnFamily.NAME);
        for (Entry<Key, Value> entry : mscanner) {
            if (BulkFileColumnFamily.getBulkLoadTid(entry.getValue()) == tid) {
                Path loadedFile = new Path(TabletFileUtil.validate(entry.getKey().getColumnQualifierData().toString()));
                if (failures.remove(loadedFile)) {
                    loadedFailures.add(loadedFile);
                }
            }
        }
    }
    // move failed files that were not loaded
    for (Path orig : failures) {
        Path dest = new Path(error, orig.getName());
        fs.rename(orig, dest);
        log.debug(FateTxId.formatTid(tid) + " renamed " + orig + " to " + dest + ": import failed");
    }
    if (!loadedFailures.isEmpty()) {
        DistributedWorkQueue bifCopyQueue = new DistributedWorkQueue(Constants.ZROOT + "/" + manager.getInstanceID() + Constants.ZBULK_FAILED_COPYQ, manager.getConfiguration(), manager.getContext());
        HashSet<String> workIds = new HashSet<>();
        for (Path orig : loadedFailures) {
            Path dest = new Path(error, orig.getName());
            if (fs.exists(dest))
                continue;
            bifCopyQueue.addWork(orig.getName(), (orig + "," + dest).getBytes(UTF_8));
            workIds.add(orig.getName());
            log.debug(FateTxId.formatTid(tid) + " added to copyq: " + orig + " to " + dest + ": failed");
        }
        bifCopyQueue.waitUntilDone(workIds);
    }
    fs.deleteRecursively(new Path(error, BulkImport.FAILURES_TXT));
    return new CleanUpBulkImport(tableId, source, bulk, error);
}