Examples with TKeyExtent org.apache.accumulo.core.dataImpl.thrift.TKeyExtent used on opensource projects

Example 16 with TKeyExtent

use of org.apache.accumulo.core.dataImpl.thrift.TKeyExtent in project accumulo by apache.

the class ThriftClientHandler method startScan.

@Override
public InitialScan startScan(TInfo tinfo, TCredentials credentials, TKeyExtent textent, TRange range, List<TColumn> columns, int batchSize, List<IterInfo> ssiList, Map<String, Map<String, String>> ssio, List<ByteBuffer> authorizations, boolean waitForWrites, boolean isolated, long readaheadThreshold, TSamplerConfiguration tSamplerConfig, long batchTimeOut, String contextArg, Map<String, String> executionHints) throws NotServingTabletException, ThriftSecurityException, org.apache.accumulo.core.tabletserver.thrift.TooManyFilesException, TSampleNotPresentException {
    TableId tableId = TableId.of(new String(textent.getTable(), UTF_8));
    NamespaceId namespaceId;
    try {
        namespaceId = server.getContext().getNamespaceId(tableId);
    } catch (TableNotFoundException e1) {
        throw new NotServingTabletException(textent);
    }
    if (!security.canScan(credentials, tableId, namespaceId, range, columns, ssiList, ssio, authorizations)) {
        throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
    }
    if (!security.authenticatedUserHasAuthorizations(credentials, authorizations)) {
        throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.BAD_AUTHORIZATIONS);
    }
    final KeyExtent extent = KeyExtent.fromThrift(textent);
    // metadata
    if (waitForWrites) {
        writeTracker.waitForWrites(TabletType.type(extent));
    }
    Tablet tablet = server.getOnlineTablet(extent);
    if (tablet == null) {
        throw new NotServingTabletException(textent);
    }
    HashSet<Column> columnSet = new HashSet<>();
    for (TColumn tcolumn : columns) {
        columnSet.add(new Column(tcolumn));
    }
    ScanParameters scanParams = new ScanParameters(batchSize, new Authorizations(authorizations), columnSet, ssiList, ssio, isolated, SamplerConfigurationImpl.fromThrift(tSamplerConfig), batchTimeOut, contextArg);
    final SingleScanSession scanSession = new SingleScanSession(credentials, extent, scanParams, readaheadThreshold, executionHints);
    scanSession.scanner = tablet.createScanner(new Range(range), scanParams, scanSession.interruptFlag);
    long sid = server.sessionManager.createSession(scanSession, true);
    ScanResult scanResult;
    try {
        scanResult = continueScan(tinfo, sid, scanSession);
    } catch (NoSuchScanIDException e) {
        log.error("The impossible happened", e);
        throw new RuntimeException();
    } finally {
        server.sessionManager.unreserveSession(sid);
    }
    return new InitialScan(sid, scanResult);
}

Example 17 with TKeyExtent

use of org.apache.accumulo.core.dataImpl.thrift.TKeyExtent in project accumulo by apache.

the class ThriftClientHandler method startMultiScan.

@Override
public InitialMultiScan startMultiScan(TInfo tinfo, TCredentials credentials, Map<TKeyExtent, List<TRange>> tbatch, List<TColumn> tcolumns, List<IterInfo> ssiList, Map<String, Map<String, String>> ssio, List<ByteBuffer> authorizations, boolean waitForWrites, TSamplerConfiguration tSamplerConfig, long batchTimeOut, String contextArg, Map<String, String> executionHints) throws ThriftSecurityException, TSampleNotPresentException {
    // find all of the tables that need to be scanned
    final HashSet<TableId> tables = new HashSet<>();
    for (TKeyExtent keyExtent : tbatch.keySet()) {
        tables.add(TableId.of(new String(keyExtent.getTable(), UTF_8)));
    }
    if (tables.size() != 1) {
        throw new IllegalArgumentException("Cannot batch scan over multiple tables");
    }
    // check if user has permission to the tables
    for (TableId tableId : tables) {
        NamespaceId namespaceId = getNamespaceId(credentials, tableId);
        if (!security.canScan(credentials, tableId, namespaceId, tbatch, tcolumns, ssiList, ssio, authorizations)) {
            throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
        }
    }
    try {
        if (!security.authenticatedUserHasAuthorizations(credentials, authorizations)) {
            throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.BAD_AUTHORIZATIONS);
        }
    } catch (ThriftSecurityException tse) {
        log.error("{} is not authorized", credentials.getPrincipal(), tse);
        throw tse;
    }
    // @formatter:off
    Map<KeyExtent, List<Range>> batch = tbatch.entrySet().stream().collect(Collectors.toMap(entry -> KeyExtent.fromThrift(entry.getKey()), entry -> entry.getValue().stream().map(Range::new).collect(Collectors.toList())));
    // @formatter:on
    // This is used to determine which thread pool to use
    KeyExtent threadPoolExtent = batch.keySet().iterator().next();
    if (waitForWrites) {
        writeTracker.waitForWrites(TabletType.type(batch.keySet()));
    }
    Set<Column> columnSet = tcolumns.isEmpty() ? Collections.emptySet() : new HashSet<>(Collections2.transform(tcolumns, Column::new));
    ScanParameters scanParams = new ScanParameters(-1, new Authorizations(authorizations), columnSet, ssiList, ssio, false, SamplerConfigurationImpl.fromThrift(tSamplerConfig), batchTimeOut, contextArg);
    final MultiScanSession mss = new MultiScanSession(credentials, threadPoolExtent, batch, scanParams, executionHints);
    mss.numTablets = batch.size();
    for (List<Range> ranges : batch.values()) {
        mss.numRanges += ranges.size();
    }
    long sid = server.sessionManager.createSession(mss, true);
    MultiScanResult result;
    try {
        result = continueMultiScan(sid, mss);
    } finally {
        server.sessionManager.unreserveSession(sid);
    }
    return new InitialMultiScan(sid, result);
}

Example 18 with TKeyExtent

use of org.apache.accumulo.core.dataImpl.thrift.TKeyExtent in project accumulo by apache.

the class ThriftClientHandler method chop.

@Override
public void chop(TInfo tinfo, TCredentials credentials, String lock, TKeyExtent textent) {
    try {
        checkPermission(credentials, lock, "chop");
    } catch (ThriftSecurityException e) {
        log.error("Caller doesn't have permission to chop extent", e);
        throw new RuntimeException(e);
    }
    KeyExtent ke = KeyExtent.fromThrift(textent);
    Tablet tablet = server.getOnlineTablet(ke);
    if (tablet != null) {
        tablet.chopFiles();
    }
}

Example 19 with TKeyExtent

use of org.apache.accumulo.core.dataImpl.thrift.TKeyExtent in project accumulo by apache.

the class ThriftClientHandler method splitTablet.

@Override
public void splitTablet(TInfo tinfo, TCredentials credentials, TKeyExtent tkeyExtent, ByteBuffer splitPoint) throws NotServingTabletException, ThriftSecurityException {
    TableId tableId = TableId.of(new String(ByteBufferUtil.toBytes(tkeyExtent.table)));
    NamespaceId namespaceId = getNamespaceId(credentials, tableId);
    if (!security.canSplitTablet(credentials, tableId, namespaceId)) {
        throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
    }
    KeyExtent keyExtent = KeyExtent.fromThrift(tkeyExtent);
    Tablet tablet = server.getOnlineTablet(keyExtent);
    if (tablet == null) {
        throw new NotServingTabletException(tkeyExtent);
    }
    if (keyExtent.endRow() == null || !keyExtent.endRow().equals(ByteBufferUtil.toText(splitPoint))) {
        try {
            if (server.splitTablet(tablet, ByteBufferUtil.toBytes(splitPoint)) == null) {
                throw new NotServingTabletException(tkeyExtent);
            }
        } catch (IOException e) {
            log.warn("Failed to split " + keyExtent, e);
            throw new RuntimeException(e);
        }
    }
}

Example 20 with TKeyExtent

use of org.apache.accumulo.core.dataImpl.thrift.TKeyExtent in project accumulo by apache.

the class LookupTask method run.

@Override
public void run() {
    MultiScanSession session = (MultiScanSession) server.getSession(scanID);
    String oldThreadName = Thread.currentThread().getName();
    try {
        if (isCancelled() || session == null)
            return;
        TableConfiguration acuTableConf = server.getTableConfiguration(session.threadPoolExtent);
        long maxResultsSize = acuTableConf.getAsBytes(Property.TABLE_SCAN_MAXMEM);
        runState.set(ScanRunState.RUNNING);
        Thread.currentThread().setName("Client: " + session.client + " User: " + session.getUser() + " Start: " + session.startTime + " Table: ");
        long bytesAdded = 0;
        long maxScanTime = 4000;
        long startTime = System.currentTimeMillis();
        List<KVEntry> results = new ArrayList<>();
        Map<KeyExtent, List<Range>> failures = new HashMap<>();
        List<KeyExtent> fullScans = new ArrayList<>();
        KeyExtent partScan = null;
        Key partNextKey = null;
        boolean partNextKeyInclusive = false;
        Iterator<Entry<KeyExtent, List<Range>>> iter = session.queries.entrySet().iterator();
        // check the time so that the read ahead thread is not monopolized
        while (iter.hasNext() && bytesAdded < maxResultsSize && (System.currentTimeMillis() - startTime) < maxScanTime) {
            Entry<KeyExtent, List<Range>> entry = iter.next();
            iter.remove();
            // check that tablet server is serving requested tablet
            Tablet tablet = server.getOnlineTablet(entry.getKey());
            if (tablet == null) {
                failures.put(entry.getKey(), entry.getValue());
                continue;
            }
            Thread.currentThread().setName("Client: " + session.client + " User: " + session.getUser() + " Start: " + session.startTime + " Tablet: " + entry.getKey());
            LookupResult lookupResult;
            try {
                // canceled
                if (isCancelled())
                    interruptFlag.set(true);
                lookupResult = tablet.lookup(entry.getValue(), results, session.scanParams, maxResultsSize - bytesAdded, interruptFlag);
                // if the tablet was closed it it possible that the
                // interrupt flag was set.... do not want it set for
                // the next
                // lookup
                interruptFlag.set(false);
            } catch (IOException e) {
                log.warn("lookup failed for tablet " + entry.getKey(), e);
                throw new RuntimeException(e);
            }
            bytesAdded += lookupResult.bytesAdded;
            if (lookupResult.unfinishedRanges.isEmpty()) {
                fullScans.add(entry.getKey());
            } else {
                if (lookupResult.closed) {
                    failures.put(entry.getKey(), lookupResult.unfinishedRanges);
                } else {
                    session.queries.put(entry.getKey(), lookupResult.unfinishedRanges);
                    partScan = entry.getKey();
                    partNextKey = lookupResult.unfinishedRanges.get(0).getStartKey();
                    partNextKeyInclusive = lookupResult.unfinishedRanges.get(0).isStartKeyInclusive();
                }
            }
        }
        long finishTime = System.currentTimeMillis();
        session.totalLookupTime += (finishTime - startTime);
        session.numEntries += results.size();
        // convert everything to thrift before adding result
        List<TKeyValue> retResults = new ArrayList<>();
        for (KVEntry entry : results) retResults.add(new TKeyValue(entry.getKey().toThrift(), ByteBuffer.wrap(entry.getValue().get())));
        // @formatter:off
        Map<TKeyExtent, List<TRange>> retFailures = failures.entrySet().stream().collect(Collectors.toMap(entry -> entry.getKey().toThrift(), entry -> entry.getValue().stream().map(Range::toThrift).collect(Collectors.toList())));
        // @formatter:on
        List<TKeyExtent> retFullScans = fullScans.stream().map(KeyExtent::toThrift).collect(Collectors.toList());
        TKeyExtent retPartScan = null;
        TKey retPartNextKey = null;
        if (partScan != null) {
            retPartScan = partScan.toThrift();
            retPartNextKey = partNextKey.toThrift();
        }
        // add results to queue
        addResult(new MultiScanResult(retResults, retFailures, retFullScans, retPartScan, retPartNextKey, partNextKeyInclusive, !session.queries.isEmpty()));
    } catch (IterationInterruptedException iie) {
        if (!isCancelled()) {
            log.warn("Iteration interrupted, when scan not cancelled", iie);
            addResult(iie);
        }
    } catch (SampleNotPresentException e) {
        addResult(e);
    } catch (Exception e) {
        log.warn("exception while doing multi-scan ", e);
        addResult(e);
    } finally {
        Thread.currentThread().setName(oldThreadName);
        runState.set(ScanRunState.FINISHED);
    }
}