Examples with ThriftTableOperationException org.apache.accumulo.core.client.impl.thrift.ThriftTableOperationException used on opensource projects

Example 1 with ThriftTableOperationException

use of org.apache.accumulo.core.client.impl.thrift.ThriftTableOperationException in project accumulo by apache.

the class ReplicationOperationsImplIT method getReplicationOperations.

/**
 * Spoof out the Master so we can call the implementation without starting a full instance.
 */
private ReplicationOperationsImpl getReplicationOperations() throws Exception {
    Master master = EasyMock.createMock(Master.class);
    EasyMock.expect(master.getConnector()).andReturn(conn).anyTimes();
    EasyMock.expect(master.getInstance()).andReturn(inst).anyTimes();
    EasyMock.replay(master);
    final MasterClientServiceHandler mcsh = new MasterClientServiceHandler(master) {

        @Override
        protected Table.ID getTableId(Instance inst, String tableName) throws ThriftTableOperationException {
            try {
                return Table.ID.of(conn.tableOperations().tableIdMap().get(tableName));
            } catch (Exception e) {
                throw new RuntimeException(e);
            }
        }
    };
    ClientContext context = new ClientContext(inst, new Credentials("root", new PasswordToken(ROOT_PASSWORD)), getClientConfig());
    return new ReplicationOperationsImpl(context) {

        @Override
        protected boolean getMasterDrain(final TInfo tinfo, final TCredentials rpcCreds, final String tableName, final Set<String> wals) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
            try {
                return mcsh.drainReplicationTable(tinfo, rpcCreds, tableName, wals);
            } catch (TException e) {
                throw new RuntimeException(e);
            }
        }
    };
}

Example 2 with ThriftTableOperationException

use of org.apache.accumulo.core.client.impl.thrift.ThriftTableOperationException in project accumulo by apache.

the class BulkImporter method importFiles.

public AssignmentStats importFiles(List<String> files, Path failureDir) throws IOException, AccumuloException, AccumuloSecurityException, ThriftTableOperationException {
    int numThreads = context.getConfiguration().getCount(Property.TSERV_BULK_PROCESS_THREADS);
    int numAssignThreads = context.getConfiguration().getCount(Property.TSERV_BULK_ASSIGNMENT_THREADS);
    timer = new StopWatch<>(Timers.class);
    timer.start(Timers.TOTAL);
    Configuration conf = CachedConfiguration.getInstance();
    VolumeManagerImpl.get(context.getConfiguration());
    final VolumeManager fs = VolumeManagerImpl.get(context.getConfiguration());
    Set<Path> paths = new HashSet<>();
    for (String file : files) {
        paths.add(new Path(file));
    }
    AssignmentStats assignmentStats = new AssignmentStats(paths.size());
    final Map<Path, List<KeyExtent>> completeFailures = Collections.synchronizedSortedMap(new TreeMap<Path, List<KeyExtent>>());
    ClientService.Client client = null;
    final TabletLocator locator = TabletLocator.getLocator(context, Table.ID.of(tableId));
    try {
        final Map<Path, List<TabletLocation>> assignments = Collections.synchronizedSortedMap(new TreeMap<Path, List<TabletLocation>>());
        timer.start(Timers.EXAMINE_MAP_FILES);
        ExecutorService threadPool = Executors.newFixedThreadPool(numThreads, new NamingThreadFactory("findOverlapping"));
        for (Path path : paths) {
            final Path mapFile = path;
            Runnable getAssignments = new Runnable() {

                @Override
                public void run() {
                    List<TabletLocation> tabletsToAssignMapFileTo = Collections.emptyList();
                    try {
                        tabletsToAssignMapFileTo = findOverlappingTablets(context, fs, locator, mapFile);
                    } catch (Exception ex) {
                        log.warn("Unable to find tablets that overlap file " + mapFile.toString(), ex);
                    }
                    log.debug("Map file {} found to overlap {} tablets", mapFile, tabletsToAssignMapFileTo.size());
                    if (tabletsToAssignMapFileTo.size() == 0) {
                        List<KeyExtent> empty = Collections.emptyList();
                        completeFailures.put(mapFile, empty);
                    } else
                        assignments.put(mapFile, tabletsToAssignMapFileTo);
                }
            };
            threadPool.submit(new TraceRunnable(new LoggingRunnable(log, getAssignments)));
        }
        threadPool.shutdown();
        while (!threadPool.isTerminated()) {
            try {
                threadPool.awaitTermination(60, TimeUnit.SECONDS);
            } catch (InterruptedException e) {
                throw new RuntimeException(e);
            }
        }
        timer.stop(Timers.EXAMINE_MAP_FILES);
        assignmentStats.attemptingAssignments(assignments);
        Map<Path, List<KeyExtent>> assignmentFailures = assignMapFiles(context, conf, fs, tableId, assignments, paths, numAssignThreads, numThreads);
        assignmentStats.assignmentsFailed(assignmentFailures);
        Map<Path, Integer> failureCount = new TreeMap<>();
        for (Entry<Path, List<KeyExtent>> entry : assignmentFailures.entrySet()) failureCount.put(entry.getKey(), 1);
        long sleepTime = 2 * 1000;
        while (assignmentFailures.size() > 0) {
            sleepTime = Math.min(sleepTime * 2, 60 * 1000);
            locator.invalidateCache();
            // assumption about assignment failures is that it caused by a split
            // happening or a missing location
            // 
            // for splits we need to find children key extents that cover the
            // same key range and are contiguous (no holes, no overlap)
            timer.start(Timers.SLEEP);
            sleepUninterruptibly(sleepTime, TimeUnit.MILLISECONDS);
            timer.stop(Timers.SLEEP);
            log.debug("Trying to assign {} map files that previously failed on some key extents", assignmentFailures.size());
            assignments.clear();
            // assign to
            for (Entry<Path, List<KeyExtent>> entry : assignmentFailures.entrySet()) {
                Iterator<KeyExtent> keListIter = entry.getValue().iterator();
                List<TabletLocation> tabletsToAssignMapFileTo = new ArrayList<>();
                while (keListIter.hasNext()) {
                    KeyExtent ke = keListIter.next();
                    timer.start(Timers.QUERY_METADATA);
                    try {
                        tabletsToAssignMapFileTo.addAll(findOverlappingTablets(context, fs, locator, entry.getKey(), ke));
                        keListIter.remove();
                    } catch (Exception ex) {
                        log.warn("Exception finding overlapping tablets, will retry tablet " + ke, ex);
                    }
                    timer.stop(Timers.QUERY_METADATA);
                }
                if (tabletsToAssignMapFileTo.size() > 0)
                    assignments.put(entry.getKey(), tabletsToAssignMapFileTo);
            }
            assignmentStats.attemptingAssignments(assignments);
            Map<Path, List<KeyExtent>> assignmentFailures2 = assignMapFiles(context, conf, fs, tableId, assignments, paths, numAssignThreads, numThreads);
            assignmentStats.assignmentsFailed(assignmentFailures2);
            // merge assignmentFailures2 into assignmentFailures
            for (Entry<Path, List<KeyExtent>> entry : assignmentFailures2.entrySet()) {
                assignmentFailures.get(entry.getKey()).addAll(entry.getValue());
                Integer fc = failureCount.get(entry.getKey());
                if (fc == null)
                    fc = 0;
                failureCount.put(entry.getKey(), fc + 1);
            }
            // remove map files that have no more key extents to assign
            Iterator<Entry<Path, List<KeyExtent>>> afIter = assignmentFailures.entrySet().iterator();
            while (afIter.hasNext()) {
                Entry<Path, List<KeyExtent>> entry = afIter.next();
                if (entry.getValue().size() == 0)
                    afIter.remove();
            }
            Set<Entry<Path, Integer>> failureIter = failureCount.entrySet();
            for (Entry<Path, Integer> entry : failureIter) {
                int retries = context.getConfiguration().getCount(Property.TSERV_BULK_RETRY);
                if (entry.getValue() > retries && assignmentFailures.get(entry.getKey()) != null) {
                    log.error("Map file {} failed more than {} times, giving up.", entry.getKey(), retries);
                    completeFailures.put(entry.getKey(), assignmentFailures.get(entry.getKey()));
                    assignmentFailures.remove(entry.getKey());
                }
            }
        }
        assignmentStats.assignmentsAbandoned(completeFailures);
        Set<Path> failedFailures = processFailures(completeFailures);
        assignmentStats.unrecoveredMapFiles(failedFailures);
        timer.stop(Timers.TOTAL);
        printReport(paths);
        return assignmentStats;
    } finally {
        if (client != null) {
            ServerClient.close(client);
        }
    }
}

Example 3 with ThriftTableOperationException

use of org.apache.accumulo.core.client.impl.thrift.ThriftTableOperationException in project accumulo by apache.

the class FateServiceHandler method waitForFateOperation.

@Override
public String waitForFateOperation(TInfo tinfo, TCredentials credentials, long opid) throws ThriftSecurityException, ThriftTableOperationException {
    authenticate(credentials);
    TStatus status = master.fate.waitForCompletion(opid);
    if (status == TStatus.FAILED) {
        Exception e = master.fate.getException(opid);
        if (e instanceof ThriftTableOperationException)
            throw (ThriftTableOperationException) e;
        else if (e instanceof ThriftSecurityException)
            throw (ThriftSecurityException) e;
        else if (e instanceof RuntimeException)
            throw (RuntimeException) e;
        else
            throw new RuntimeException(e);
    }
    String ret = master.fate.getReturn(opid);
    if (ret == null)
        // thrift does not like returning null
        ret = "";
    return ret;
}

Example 4 with ThriftTableOperationException

use of org.apache.accumulo.core.client.impl.thrift.ThriftTableOperationException in project accumulo by apache.

the class MasterClientServiceHandler method waitForFlush.

@Override
public void waitForFlush(TInfo tinfo, TCredentials c, String tableIdStr, ByteBuffer startRow, ByteBuffer endRow, long flushID, long maxLoops) throws ThriftSecurityException, ThriftTableOperationException {
    Table.ID tableId = Table.ID.of(tableIdStr);
    Namespace.ID namespaceId = getNamespaceIdFromTableId(TableOperation.FLUSH, tableId);
    master.security.canFlush(c, tableId, namespaceId);
    if (endRow != null && startRow != null && ByteBufferUtil.toText(startRow).compareTo(ByteBufferUtil.toText(endRow)) >= 0)
        throw new ThriftTableOperationException(tableId.canonicalID(), null, TableOperation.FLUSH, TableOperationExceptionType.BAD_RANGE, "start row must be less than end row");
    Set<TServerInstance> serversToFlush = new HashSet<>(master.tserverSet.getCurrentServers());
    for (long l = 0; l < maxLoops; l++) {
        for (TServerInstance instance : serversToFlush) {
            try {
                final TServerConnection server = master.tserverSet.getConnection(instance);
                if (server != null)
                    server.flush(master.masterLock, tableId, ByteBufferUtil.toBytes(startRow), ByteBufferUtil.toBytes(endRow));
            } catch (TException ex) {
                Master.log.error(ex.toString());
            }
        }
        if (l == maxLoops - 1)
            break;
        sleepUninterruptibly(50, TimeUnit.MILLISECONDS);
        serversToFlush.clear();
        try {
            Connector conn = master.getConnector();
            Scanner scanner;
            if (tableId.equals(MetadataTable.ID)) {
                scanner = new IsolatedScanner(conn.createScanner(RootTable.NAME, Authorizations.EMPTY));
                scanner.setRange(MetadataSchema.TabletsSection.getRange());
            } else {
                scanner = new IsolatedScanner(conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY));
                Range range = new KeyExtent(tableId, null, ByteBufferUtil.toText(startRow)).toMetadataRange();
                scanner.setRange(range.clip(MetadataSchema.TabletsSection.getRange()));
            }
            TabletsSection.ServerColumnFamily.FLUSH_COLUMN.fetch(scanner);
            TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.fetch(scanner);
            scanner.fetchColumnFamily(TabletsSection.CurrentLocationColumnFamily.NAME);
            scanner.fetchColumnFamily(LogColumnFamily.NAME);
            RowIterator ri = new RowIterator(scanner);
            int tabletsToWaitFor = 0;
            int tabletCount = 0;
            Text ert = ByteBufferUtil.toText(endRow);
            while (ri.hasNext()) {
                Iterator<Entry<Key, Value>> row = ri.next();
                long tabletFlushID = -1;
                int logs = 0;
                boolean online = false;
                TServerInstance server = null;
                Entry<Key, Value> entry = null;
                while (row.hasNext()) {
                    entry = row.next();
                    Key key = entry.getKey();
                    if (TabletsSection.ServerColumnFamily.FLUSH_COLUMN.equals(key.getColumnFamily(), key.getColumnQualifier())) {
                        tabletFlushID = Long.parseLong(entry.getValue().toString());
                    }
                    if (LogColumnFamily.NAME.equals(key.getColumnFamily()))
                        logs++;
                    if (TabletsSection.CurrentLocationColumnFamily.NAME.equals(key.getColumnFamily())) {
                        online = true;
                        server = new TServerInstance(entry.getValue(), key.getColumnQualifier());
                    }
                }
                // when tablet is not online and has no logs, there is no reason to wait for it
                if ((online || logs > 0) && tabletFlushID < flushID) {
                    tabletsToWaitFor++;
                    if (server != null)
                        serversToFlush.add(server);
                }
                tabletCount++;
                Text tabletEndRow = new KeyExtent(entry.getKey().getRow(), (Text) null).getEndRow();
                if (tabletEndRow == null || (ert != null && tabletEndRow.compareTo(ert) >= 0))
                    break;
            }
            if (tabletsToWaitFor == 0)
                break;
            if (tabletCount == 0 && !Tables.exists(master.getInstance(), tableId))
                throw new ThriftTableOperationException(tableId.canonicalID(), null, TableOperation.FLUSH, TableOperationExceptionType.NOTFOUND, null);
        } catch (AccumuloException | TabletDeletedException e) {
            Master.log.debug("Failed to scan {} table to wait for flush {}", MetadataTable.NAME, tableId, e);
        } catch (AccumuloSecurityException e) {
            Master.log.warn("{}", e.getMessage(), e);
            throw new ThriftSecurityException();
        } catch (TableNotFoundException e) {
            Master.log.error("{}", e.getMessage(), e);
            throw new ThriftTableOperationException();
        }
    }
}

Example 5 with ThriftTableOperationException

use of org.apache.accumulo.core.client.impl.thrift.ThriftTableOperationException in project accumulo by apache.

the class MasterClientServiceHandler method alterNamespaceProperty.

private void alterNamespaceProperty(TCredentials c, String namespace, String property, String value, TableOperation op) throws ThriftSecurityException, ThriftTableOperationException {
    Namespace.ID namespaceId = null;
    namespaceId = ClientServiceHandler.checkNamespaceId(master.getInstance(), namespace, op);
    if (!master.security.canAlterNamespace(c, namespaceId))
        throw new ThriftSecurityException(c.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
    try {
        if (value == null) {
            NamespacePropUtil.removeNamespaceProperty(namespaceId, property);
        } else {
            NamespacePropUtil.setNamespaceProperty(namespaceId, property, value);
        }
    } catch (KeeperException.NoNodeException e) {
        // race condition... namespace no longer exists? This call will throw an exception if the namespace was deleted:
        ClientServiceHandler.checkNamespaceId(master.getInstance(), namespace, op);
        log.info("Error altering namespace property", e);
        throw new ThriftTableOperationException(namespaceId.canonicalID(), namespace, op, TableOperationExceptionType.OTHER, "Problem altering namespaceproperty");
    } catch (Exception e) {
        log.error("Problem altering namespace property", e);
        throw new ThriftTableOperationException(namespaceId.canonicalID(), namespace, op, TableOperationExceptionType.OTHER, "Problem altering namespace property");
    }
}