Examples with AcceptableThriftTableOperationException org.apache.accumulo.core.clientImpl.AcceptableThriftTableOperationException used on opensource projects

Example 1 with AcceptableThriftTableOperationException

use of org.apache.accumulo.core.clientImpl.AcceptableThriftTableOperationException in project accumulo by apache.

the class PrepBulkImport method sanityCheckLoadMapping.

/**
 * Checks a load mapping to ensure all of the rows in the mapping exists in the table and that no
 * file goes to too many tablets.
 */
@VisibleForTesting
static void sanityCheckLoadMapping(String tableId, LoadMappingIterator lmi, TabletIterFactory tabletIterFactory, int maxNumTablets, long tid) throws Exception {
    var currRange = lmi.next();
    Text startRow = currRange.getKey().prevEndRow();
    Iterator<KeyExtent> tabletIter = tabletIterFactory.newTabletIter(startRow);
    KeyExtent currTablet = tabletIter.next();
    var fileCounts = new HashMap<String, Integer>();
    int count;
    if (!tabletIter.hasNext() && equals(KeyExtent::prevEndRow, currTablet, currRange.getKey()) && equals(KeyExtent::endRow, currTablet, currRange.getKey()))
        currRange = null;
    while (tabletIter.hasNext()) {
        if (currRange == null) {
            if (!lmi.hasNext()) {
                break;
            }
            currRange = lmi.next();
        }
        while (!equals(KeyExtent::prevEndRow, currTablet, currRange.getKey()) && tabletIter.hasNext()) {
            currTablet = tabletIter.next();
        }
        boolean matchedPrevRow = equals(KeyExtent::prevEndRow, currTablet, currRange.getKey());
        count = matchedPrevRow ? 1 : 0;
        while (!equals(KeyExtent::endRow, currTablet, currRange.getKey()) && tabletIter.hasNext()) {
            currTablet = tabletIter.next();
            count++;
        }
        if (!matchedPrevRow || !equals(KeyExtent::endRow, currTablet, currRange.getKey())) {
            break;
        }
        if (maxNumTablets > 0) {
            int fc = count;
            currRange.getValue().forEach(fileInfo -> fileCounts.merge(fileInfo.getFileName(), fc, Integer::sum));
        }
        currRange = null;
    }
    if (currRange != null || lmi.hasNext()) {
        // merge happened after the mapping was generated and before the table lock was acquired
        throw new AcceptableThriftTableOperationException(tableId, null, TableOperation.BULK_IMPORT, TableOperationExceptionType.BULK_CONCURRENT_MERGE, "Concurrent merge happened");
    }
    if (maxNumTablets > 0) {
        fileCounts.values().removeIf(c -> c <= maxNumTablets);
        if (!fileCounts.isEmpty()) {
            throw new AcceptableThriftTableOperationException(tableId, null, TableOperation.BULK_IMPORT, TableOperationExceptionType.OTHER, "Files overlap the configured max (" + maxNumTablets + ") number of tablets: " + new TreeMap<>(fileCounts));
        }
    }
}

Example 2 with AcceptableThriftTableOperationException

use of org.apache.accumulo.core.clientImpl.AcceptableThriftTableOperationException in project accumulo by apache.

the class BulkImport method call.

@Override
public Repo<Manager> call(long tid, Manager manager) throws Exception {
    String fmtTid = FateTxId.formatTid(tid);
    log.debug(" {} sourceDir {}", fmtTid, sourceDir);
    Utils.getReadLock(manager, tableId, tid).lock();
    // check that the error directory exists and is empty
    VolumeManager fs = manager.getVolumeManager();
    Path errorPath = new Path(errorDir);
    FileStatus errorStatus = null;
    try {
        errorStatus = fs.getFileStatus(errorPath);
    } catch (FileNotFoundException ex) {
    // ignored
    }
    if (errorStatus == null)
        throw new AcceptableThriftTableOperationException(tableId.canonical(), null, TableOperation.BULK_IMPORT, TableOperationExceptionType.BULK_BAD_ERROR_DIRECTORY, errorDir + " does not exist");
    if (!errorStatus.isDirectory())
        throw new AcceptableThriftTableOperationException(tableId.canonical(), null, TableOperation.BULK_IMPORT, TableOperationExceptionType.BULK_BAD_ERROR_DIRECTORY, errorDir + " is not a directory");
    if (fs.listStatus(errorPath).length != 0)
        throw new AcceptableThriftTableOperationException(tableId.canonical(), null, TableOperation.BULK_IMPORT, TableOperationExceptionType.BULK_BAD_ERROR_DIRECTORY, errorDir + " is not empty");
    ZooArbitrator.start(manager.getContext(), Constants.BULK_ARBITRATOR_TYPE, tid);
    manager.updateBulkImportStatus(sourceDir, BulkImportState.MOVING);
    // move the files into the directory
    try {
        String bulkDir = prepareBulkImport(manager.getContext(), fs, sourceDir, tableId, tid);
        log.debug(" {} bulkDir {}", tid, bulkDir);
        return new LoadFiles(tableId, sourceDir, bulkDir, errorDir, setTime);
    } catch (IOException ex) {
        log.error("error preparing the bulk import directory", ex);
        throw new AcceptableThriftTableOperationException(tableId.canonical(), null, TableOperation.BULK_IMPORT, TableOperationExceptionType.BULK_BAD_INPUT_DIRECTORY, sourceDir + ": " + ex);
    }
}

Example 3 with AcceptableThriftTableOperationException

use of org.apache.accumulo.core.clientImpl.AcceptableThriftTableOperationException in project accumulo by apache.

the class LoadFiles method call.

@Override
public Repo<Manager> call(final long tid, final Manager manager) throws Exception {
    manager.updateBulkImportStatus(source, BulkImportState.LOADING);
    ExecutorService executor = getThreadPool(manager);
    final AccumuloConfiguration conf = manager.getConfiguration();
    VolumeManager fs = manager.getVolumeManager();
    List<FileStatus> files = new ArrayList<>();
    Collections.addAll(files, fs.listStatus(new Path(bulk)));
    log.debug(FateTxId.formatTid(tid) + " importing " + files.size() + " files");
    Path writable = new Path(this.errorDir, ".iswritable");
    if (!fs.createNewFile(writable)) {
        // Maybe this is a re-try... clear the flag and try again
        fs.delete(writable);
        if (!fs.createNewFile(writable))
            throw new AcceptableThriftTableOperationException(tableId.canonical(), null, TableOperation.BULK_IMPORT, TableOperationExceptionType.BULK_BAD_ERROR_DIRECTORY, "Unable to write to " + this.errorDir);
    }
    fs.delete(writable);
    final Set<String> filesToLoad = Collections.synchronizedSet(new HashSet<>());
    for (FileStatus f : files) filesToLoad.add(f.getPath().toString());
    final int RETRIES = Math.max(1, conf.getCount(Property.MANAGER_BULK_RETRIES));
    for (int attempt = 0; attempt < RETRIES && !filesToLoad.isEmpty(); attempt++) {
        List<Future<Void>> results = new ArrayList<>();
        if (manager.onlineTabletServers().isEmpty())
            log.warn("There are no tablet server to process bulk import, waiting (tid = " + FateTxId.formatTid(tid) + ")");
        while (manager.onlineTabletServers().isEmpty()) {
            sleepUninterruptibly(500, TimeUnit.MILLISECONDS);
        }
        // Use the threadpool to assign files one-at-a-time to the server
        final List<String> loaded = Collections.synchronizedList(new ArrayList<>());
        final TServerInstance[] servers;
        String prop = conf.get(Property.MANAGER_BULK_TSERVER_REGEX);
        if (prop == null || "".equals(prop)) {
            servers = manager.onlineTabletServers().toArray(new TServerInstance[0]);
        } else {
            Pattern regex = Pattern.compile(prop);
            List<TServerInstance> subset = new ArrayList<>();
            manager.onlineTabletServers().forEach(t -> {
                if (regex.matcher(t.getHost()).matches()) {
                    subset.add(t);
                }
            });
            if (subset.isEmpty()) {
                log.warn("There are no tablet servers online that match supplied regex: {}", conf.get(Property.MANAGER_BULK_TSERVER_REGEX));
            }
            servers = subset.toArray(new TServerInstance[0]);
        }
        if (servers.length > 0) {
            for (final String file : filesToLoad) {
                results.add(executor.submit(() -> {
                    ClientService.Client client = null;
                    HostAndPort server = null;
                    try {
                        // get a connection to a random tablet server, do not prefer cached connections
                        // because this is running on the manager and there are lots of connections to tablet
                        // servers serving the metadata tablets
                        long timeInMillis = manager.getConfiguration().getTimeInMillis(Property.MANAGER_BULK_TIMEOUT);
                        server = servers[random.nextInt(servers.length)].getHostAndPort();
                        client = ThriftUtil.getTServerClient(server, manager.getContext(), timeInMillis);
                        List<String> attempt1 = Collections.singletonList(file);
                        log.debug("Asking " + server + " to bulk import " + file);
                        List<String> fail = client.bulkImportFiles(TraceUtil.traceInfo(), manager.getContext().rpcCreds(), tid, tableId.canonical(), attempt1, errorDir, setTime);
                        if (fail.isEmpty()) {
                            loaded.add(file);
                        }
                    } catch (Exception ex) {
                        log.error("rpc failed server:" + server + ", tid:" + FateTxId.formatTid(tid) + " " + ex);
                    } finally {
                        ThriftUtil.returnClient(client, manager.getContext());
                    }
                    return null;
                }));
            }
        }
        for (Future<Void> f : results) {
            f.get();
        }
        filesToLoad.removeAll(loaded);
        if (!filesToLoad.isEmpty()) {
            log.debug(FateTxId.formatTid(tid) + " attempt " + (attempt + 1) + " " + sampleList(filesToLoad, 10) + " failed");
            sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
        }
    }
    FSDataOutputStream failFile = fs.overwrite(new Path(errorDir, BulkImport.FAILURES_TXT));
    try (BufferedWriter out = new BufferedWriter(new OutputStreamWriter(failFile, UTF_8))) {
        for (String f : filesToLoad) {
            out.write(f);
            out.write("\n");
        }
    }
    // return the next step, which will perform cleanup
    return new CompleteBulkImport(tableId, source, bulk, errorDir);
}

Example 4 with AcceptableThriftTableOperationException

use of org.apache.accumulo.core.clientImpl.AcceptableThriftTableOperationException in project accumulo by apache.

the class CompactionDriver method isReady.

@Override
public long isReady(long tid, Manager manager) throws Exception {
    if (tableId.equals(RootTable.ID)) {
        // this codes not properly handle the root table. See #798
        return 0;
    }
    String zCancelID = createCompactionCancellationPath(manager.getInstanceID(), tableId);
    ZooReaderWriter zoo = manager.getContext().getZooReaderWriter();
    if (Long.parseLong(new String(zoo.getData(zCancelID))) >= compactId) {
        // compaction was canceled
        throw new AcceptableThriftTableOperationException(tableId.canonical(), null, TableOperation.COMPACT, TableOperationExceptionType.OTHER, TableOperationsImpl.COMPACTION_CANCELED_MSG);
    }
    String deleteMarkerPath = PreDeleteTable.createDeleteMarkerPath(manager.getInstanceID(), tableId);
    if (zoo.exists(deleteMarkerPath)) {
        // table is being deleted
        throw new AcceptableThriftTableOperationException(tableId.canonical(), null, TableOperation.COMPACT, TableOperationExceptionType.OTHER, TableOperationsImpl.TABLE_DELETED_MSG);
    }
    MapCounter<TServerInstance> serversToFlush = new MapCounter<>();
    long t1 = System.currentTimeMillis();
    int tabletsToWaitFor = 0;
    int tabletCount = 0;
    TabletsMetadata tablets = TabletsMetadata.builder(manager.getContext()).forTable(tableId).overlapping(startRow, endRow).fetch(LOCATION, PREV_ROW, COMPACT_ID).build();
    for (TabletMetadata tablet : tablets) {
        if (tablet.getCompactId().orElse(-1) < compactId) {
            tabletsToWaitFor++;
            if (tablet.hasCurrent()) {
                serversToFlush.increment(tablet.getLocation(), 1);
            }
        }
        tabletCount++;
    }
    long scanTime = System.currentTimeMillis() - t1;
    manager.getContext().clearTableListCache();
    if (tabletCount == 0 && !manager.getContext().tableNodeExists(tableId))
        throw new AcceptableThriftTableOperationException(tableId.canonical(), null, TableOperation.COMPACT, TableOperationExceptionType.NOTFOUND, null);
    if (serversToFlush.size() == 0 && manager.getContext().getTableState(tableId) == TableState.OFFLINE)
        throw new AcceptableThriftTableOperationException(tableId.canonical(), null, TableOperation.COMPACT, TableOperationExceptionType.OFFLINE, null);
    if (tabletsToWaitFor == 0)
        return 0;
    for (TServerInstance tsi : serversToFlush.keySet()) {
        try {
            final TServerConnection server = manager.getConnection(tsi);
            if (server != null)
                server.compact(manager.getManagerLock(), tableId.canonical(), startRow, endRow);
        } catch (TException ex) {
            LoggerFactory.getLogger(CompactionDriver.class).error(ex.toString());
        }
    }
    long sleepTime = 500;
    // make wait time depend on the server with the most to compact
    if (serversToFlush.size() > 0)
        sleepTime = serversToFlush.max() * sleepTime;
    sleepTime = Math.max(2 * scanTime, sleepTime);
    sleepTime = Math.min(sleepTime, 30000);
    return sleepTime;
}

Example 5 with AcceptableThriftTableOperationException

use of org.apache.accumulo.core.clientImpl.AcceptableThriftTableOperationException in project accumulo by apache.

the class PopulateMetadataTable method readMappingFile.

static void readMappingFile(VolumeManager fs, ImportedTableInfo tableInfo, String importDir, Map<String, String> fileNameMappings) throws Exception {
    try (BufferedReader in = new BufferedReader(new InputStreamReader(fs.open(new Path(importDir, "mappings.txt")), UTF_8))) {
        String line, prev;
        while ((line = in.readLine()) != null) {
            String[] sa = line.split(":", 2);
            prev = fileNameMappings.put(sa[0], importDir + "/" + sa[1]);
            if (prev != null) {
                String msg = "File exists in multiple import directories: '" + sa[0].replaceAll("[\r\n]", "") + "'";
                log.warn(msg);
                throw new AcceptableThriftTableOperationException(tableInfo.tableId.canonical(), tableInfo.tableName, TableOperation.IMPORT, TableOperationExceptionType.OTHER, msg);
            }
        }
    }
}