Search in sources :

Example 1 with ScanParameters

use of org.apache.accumulo.tserver.scan.ScanParameters in project accumulo by apache.

the class ThriftClientHandler method startScan.

@Override
public InitialScan startScan(TInfo tinfo, TCredentials credentials, TKeyExtent textent, TRange range, List<TColumn> columns, int batchSize, List<IterInfo> ssiList, Map<String, Map<String, String>> ssio, List<ByteBuffer> authorizations, boolean waitForWrites, boolean isolated, long readaheadThreshold, TSamplerConfiguration tSamplerConfig, long batchTimeOut, String contextArg, Map<String, String> executionHints) throws NotServingTabletException, ThriftSecurityException, org.apache.accumulo.core.tabletserver.thrift.TooManyFilesException, TSampleNotPresentException {
    TableId tableId = TableId.of(new String(textent.getTable(), UTF_8));
    NamespaceId namespaceId;
    try {
        namespaceId = server.getContext().getNamespaceId(tableId);
    } catch (TableNotFoundException e1) {
        throw new NotServingTabletException(textent);
    }
    if (!security.canScan(credentials, tableId, namespaceId, range, columns, ssiList, ssio, authorizations)) {
        throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
    }
    if (!security.authenticatedUserHasAuthorizations(credentials, authorizations)) {
        throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.BAD_AUTHORIZATIONS);
    }
    final KeyExtent extent = KeyExtent.fromThrift(textent);
    // metadata
    if (waitForWrites) {
        writeTracker.waitForWrites(TabletType.type(extent));
    }
    Tablet tablet = server.getOnlineTablet(extent);
    if (tablet == null) {
        throw new NotServingTabletException(textent);
    }
    HashSet<Column> columnSet = new HashSet<>();
    for (TColumn tcolumn : columns) {
        columnSet.add(new Column(tcolumn));
    }
    ScanParameters scanParams = new ScanParameters(batchSize, new Authorizations(authorizations), columnSet, ssiList, ssio, isolated, SamplerConfigurationImpl.fromThrift(tSamplerConfig), batchTimeOut, contextArg);
    final SingleScanSession scanSession = new SingleScanSession(credentials, extent, scanParams, readaheadThreshold, executionHints);
    scanSession.scanner = tablet.createScanner(new Range(range), scanParams, scanSession.interruptFlag);
    long sid = server.sessionManager.createSession(scanSession, true);
    ScanResult scanResult;
    try {
        scanResult = continueScan(tinfo, sid, scanSession);
    } catch (NoSuchScanIDException e) {
        log.error("The impossible happened", e);
        throw new RuntimeException();
    } finally {
        server.sessionManager.unreserveSession(sid);
    }
    return new InitialScan(sid, scanResult);
}
Also used : TableId(org.apache.accumulo.core.data.TableId) Authorizations(org.apache.accumulo.core.security.Authorizations) ScanResult(org.apache.accumulo.core.dataImpl.thrift.ScanResult) MultiScanResult(org.apache.accumulo.core.dataImpl.thrift.MultiScanResult) TColumn(org.apache.accumulo.core.dataImpl.thrift.TColumn) NotServingTabletException(org.apache.accumulo.core.tabletserver.thrift.NotServingTabletException) TRange(org.apache.accumulo.core.dataImpl.thrift.TRange) Range(org.apache.accumulo.core.data.Range) TRowRange(org.apache.accumulo.core.dataImpl.thrift.TRowRange) ThriftSecurityException(org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException) TKeyExtent(org.apache.accumulo.core.dataImpl.thrift.TKeyExtent) KeyExtent(org.apache.accumulo.core.dataImpl.KeyExtent) InitialScan(org.apache.accumulo.core.dataImpl.thrift.InitialScan) NoSuchScanIDException(org.apache.accumulo.core.tabletserver.thrift.NoSuchScanIDException) TableNotFoundException(org.apache.accumulo.core.client.TableNotFoundException) Column(org.apache.accumulo.core.data.Column) TColumn(org.apache.accumulo.core.dataImpl.thrift.TColumn) ScanParameters(org.apache.accumulo.tserver.scan.ScanParameters) Tablet(org.apache.accumulo.tserver.tablet.Tablet) NamespaceId(org.apache.accumulo.core.data.NamespaceId) SingleScanSession(org.apache.accumulo.tserver.session.SingleScanSession) HashSet(java.util.HashSet)

Example 2 with ScanParameters

use of org.apache.accumulo.tserver.scan.ScanParameters in project accumulo by apache.

the class ThriftClientHandler method startMultiScan.

@Override
public InitialMultiScan startMultiScan(TInfo tinfo, TCredentials credentials, Map<TKeyExtent, List<TRange>> tbatch, List<TColumn> tcolumns, List<IterInfo> ssiList, Map<String, Map<String, String>> ssio, List<ByteBuffer> authorizations, boolean waitForWrites, TSamplerConfiguration tSamplerConfig, long batchTimeOut, String contextArg, Map<String, String> executionHints) throws ThriftSecurityException, TSampleNotPresentException {
    // find all of the tables that need to be scanned
    final HashSet<TableId> tables = new HashSet<>();
    for (TKeyExtent keyExtent : tbatch.keySet()) {
        tables.add(TableId.of(new String(keyExtent.getTable(), UTF_8)));
    }
    if (tables.size() != 1) {
        throw new IllegalArgumentException("Cannot batch scan over multiple tables");
    }
    // check if user has permission to the tables
    for (TableId tableId : tables) {
        NamespaceId namespaceId = getNamespaceId(credentials, tableId);
        if (!security.canScan(credentials, tableId, namespaceId, tbatch, tcolumns, ssiList, ssio, authorizations)) {
            throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
        }
    }
    try {
        if (!security.authenticatedUserHasAuthorizations(credentials, authorizations)) {
            throw new ThriftSecurityException(credentials.getPrincipal(), SecurityErrorCode.BAD_AUTHORIZATIONS);
        }
    } catch (ThriftSecurityException tse) {
        log.error("{} is not authorized", credentials.getPrincipal(), tse);
        throw tse;
    }
    // @formatter:off
    Map<KeyExtent, List<Range>> batch = tbatch.entrySet().stream().collect(Collectors.toMap(entry -> KeyExtent.fromThrift(entry.getKey()), entry -> entry.getValue().stream().map(Range::new).collect(Collectors.toList())));
    // @formatter:on
    // This is used to determine which thread pool to use
    KeyExtent threadPoolExtent = batch.keySet().iterator().next();
    if (waitForWrites) {
        writeTracker.waitForWrites(TabletType.type(batch.keySet()));
    }
    Set<Column> columnSet = tcolumns.isEmpty() ? Collections.emptySet() : new HashSet<>(Collections2.transform(tcolumns, Column::new));
    ScanParameters scanParams = new ScanParameters(-1, new Authorizations(authorizations), columnSet, ssiList, ssio, false, SamplerConfigurationImpl.fromThrift(tSamplerConfig), batchTimeOut, contextArg);
    final MultiScanSession mss = new MultiScanSession(credentials, threadPoolExtent, batch, scanParams, executionHints);
    mss.numTablets = batch.size();
    for (List<Range> ranges : batch.values()) {
        mss.numRanges += ranges.size();
    }
    long sid = server.sessionManager.createSession(mss, true);
    MultiScanResult result;
    try {
        result = continueMultiScan(sid, mss);
    } finally {
        server.sessionManager.unreserveSession(sid);
    }
    return new InitialMultiScan(sid, result);
}
Also used : TableId(org.apache.accumulo.core.data.TableId) FileSystem(org.apache.hadoop.fs.FileSystem) TooManyFilesException(org.apache.accumulo.server.fs.TooManyFilesException) Text(org.apache.hadoop.io.Text) Collections2(com.google.common.collect.Collections2) ConstraintViolationSummary(org.apache.accumulo.core.data.ConstraintViolationSummary) TableNotFoundException(org.apache.accumulo.core.client.TableNotFoundException) Future(java.util.concurrent.Future) TSummaries(org.apache.accumulo.core.dataImpl.thrift.TSummaries) CompressedIterators(org.apache.accumulo.core.clientImpl.CompressedIterators) ExternalCompactionId(org.apache.accumulo.core.metadata.schema.ExternalCompactionId) ScanResult(org.apache.accumulo.core.dataImpl.thrift.ScanResult) Map(java.util.Map) RootTable(org.apache.accumulo.core.metadata.RootTable) MapFileInfo(org.apache.accumulo.core.dataImpl.thrift.MapFileInfo) BulkImportState(org.apache.accumulo.core.master.thrift.BulkImportState) NoSuchScanIDException(org.apache.accumulo.core.tabletserver.thrift.NoSuchScanIDException) Property(org.apache.accumulo.core.conf.Property) VolumeManager(org.apache.accumulo.server.fs.VolumeManager) CancellationException(java.util.concurrent.CancellationException) Column(org.apache.accumulo.core.data.Column) TSummaryRequest(org.apache.accumulo.core.dataImpl.thrift.TSummaryRequest) TabletStats(org.apache.accumulo.core.tabletserver.thrift.TabletStats) PreparedMutations(org.apache.accumulo.tserver.tablet.PreparedMutations) TConditionalSession(org.apache.accumulo.core.dataImpl.thrift.TConditionalSession) Set(java.util.Set) ThriftSecurityException(org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException) TCMStatus(org.apache.accumulo.core.dataImpl.thrift.TCMStatus) TCredentials(org.apache.accumulo.core.securityImpl.thrift.TCredentials) TExternalCompactionJob(org.apache.accumulo.core.tabletserver.thrift.TExternalCompactionJob) TSamplerConfiguration(org.apache.accumulo.core.tabletserver.thrift.TSamplerConfiguration) ScanDispatcher(org.apache.accumulo.core.spi.scan.ScanDispatcher) RowLock(org.apache.accumulo.tserver.RowLocks.RowLock) ServiceLock(org.apache.accumulo.fate.zookeeper.ServiceLock) UtilWaitThread.sleepUninterruptibly(org.apache.accumulo.fate.util.UtilWaitThread.sleepUninterruptibly) TRange(org.apache.accumulo.core.dataImpl.thrift.TRange) SamplerConfigurationImpl(org.apache.accumulo.core.sample.impl.SamplerConfigurationImpl) KVEntry(org.apache.accumulo.tserver.tablet.KVEntry) SummaryCollection(org.apache.accumulo.core.summary.SummaryCollection) MINUTES(java.util.concurrent.TimeUnit.MINUTES) Threads(org.apache.accumulo.core.util.threads.Threads) ZooUtil(org.apache.accumulo.fate.zookeeper.ZooUtil) DurabilityImpl(org.apache.accumulo.core.clientImpl.DurabilityImpl) ArrayList(java.util.ArrayList) MultiScanSession(org.apache.accumulo.tserver.session.MultiScanSession) TSampleNotPresentException(org.apache.accumulo.core.tabletserver.thrift.TSampleNotPresentException) Gatherer(org.apache.accumulo.core.summary.Gatherer) ActiveCompaction(org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction) LookupTask(org.apache.accumulo.tserver.scan.LookupTask) FSError(org.apache.hadoop.fs.FSError) Key(org.apache.accumulo.core.data.Key) FileSystemResolver(org.apache.accumulo.core.summary.Gatherer.FileSystemResolver) CommitSession(org.apache.accumulo.tserver.tablet.CommitSession) TabletFile(org.apache.accumulo.core.metadata.TabletFile) InitialScan(org.apache.accumulo.core.dataImpl.thrift.InitialScan) TKeyExtent(org.apache.accumulo.core.dataImpl.thrift.TKeyExtent) ServerMutation(org.apache.accumulo.server.data.ServerMutation) TConditionalMutation(org.apache.accumulo.core.dataImpl.thrift.TConditionalMutation) TColumn(org.apache.accumulo.core.dataImpl.thrift.TColumn) KeyExtent(org.apache.accumulo.core.dataImpl.KeyExtent) ConstraintViolationException(org.apache.accumulo.core.tabletserver.thrift.ConstraintViolationException) TException(org.apache.thrift.TException) IOException(java.io.IOException) Range(org.apache.accumulo.core.data.Range) ExecutionException(java.util.concurrent.ExecutionException) NoNodeException(org.apache.zookeeper.KeeperException.NoNodeException) ActiveScan(org.apache.accumulo.core.tabletserver.thrift.ActiveScan) TableId(org.apache.accumulo.core.data.TableId) FileCompactor(org.apache.accumulo.server.compaction.FileCompactor) NextBatchTask(org.apache.accumulo.tserver.scan.NextBatchTask) TabletServerStatus(org.apache.accumulo.core.master.thrift.TabletServerStatus) MultiScanResult(org.apache.accumulo.core.dataImpl.thrift.MultiScanResult) LoggerFactory(org.slf4j.LoggerFactory) TimeoutException(java.util.concurrent.TimeoutException) MetadataTable(org.apache.accumulo.core.metadata.MetadataTable) TabletLogger(org.apache.accumulo.core.logging.TabletLogger) TServerUtils(org.apache.accumulo.server.rpc.TServerUtils) Mutation(org.apache.accumulo.core.data.Mutation) TKeyValue(org.apache.accumulo.core.dataImpl.thrift.TKeyValue) IterInfo(org.apache.accumulo.core.dataImpl.thrift.IterInfo) ScanParameters(org.apache.accumulo.tserver.scan.ScanParameters) ByteBuffer(java.nio.ByteBuffer) BlockCache(org.apache.accumulo.core.spi.cache.BlockCache) ConditionalSession(org.apache.accumulo.tserver.session.ConditionalSession) TInfo(org.apache.accumulo.core.trace.thrift.TInfo) UpdateSession(org.apache.accumulo.tserver.session.UpdateSession) TCMResult(org.apache.accumulo.core.dataImpl.thrift.TCMResult) TabletClientService(org.apache.accumulo.core.tabletserver.thrift.TabletClientService) Path(org.apache.hadoop.fs.Path) TUnloadTabletGoal(org.apache.accumulo.core.tabletserver.thrift.TUnloadTabletGoal) SingleScanSession(org.apache.accumulo.tserver.session.SingleScanSession) ConditionChecker(org.apache.accumulo.tserver.ConditionCheckerContext.ConditionChecker) ServerConditionalMutation(org.apache.accumulo.tserver.data.ServerConditionalMutation) Span(io.opentelemetry.api.trace.Span) Collection(java.util.Collection) Durability(org.apache.accumulo.core.client.Durability) TransactionWatcher(org.apache.accumulo.server.zookeeper.TransactionWatcher) Collectors(java.util.stream.Collectors) TabletClosedException(org.apache.accumulo.tserver.tablet.TabletClosedException) List(java.util.List) TableConfiguration(org.apache.accumulo.server.conf.TableConfiguration) Pair(org.apache.accumulo.core.util.Pair) Entry(java.util.Map.Entry) NamespaceId(org.apache.accumulo.core.data.NamespaceId) TraceUtil(org.apache.accumulo.core.trace.TraceUtil) ByteBufferUtil(org.apache.accumulo.core.util.ByteBufferUtil) IterationInterruptedException(org.apache.accumulo.core.iteratorsImpl.system.IterationInterruptedException) TRowRange(org.apache.accumulo.core.dataImpl.thrift.TRowRange) TCompactionQueueSummary(org.apache.accumulo.core.tabletserver.thrift.TCompactionQueueSummary) HashMap(java.util.HashMap) CompactionConfig(org.apache.accumulo.core.client.admin.CompactionConfig) HashSet(java.util.HashSet) NotServingTabletException(org.apache.accumulo.core.tabletserver.thrift.NotServingTabletException) SummarySession(org.apache.accumulo.tserver.session.SummarySession) Halt(org.apache.accumulo.core.util.Halt) AccumuloSecurityException(org.apache.accumulo.core.client.AccumuloSecurityException) InitialMultiScan(org.apache.accumulo.core.dataImpl.thrift.InitialMultiScan) TableOperationExceptionType(org.apache.accumulo.core.clientImpl.thrift.TableOperationExceptionType) ExecutorService(java.util.concurrent.ExecutorService) UpdateErrors(org.apache.accumulo.core.dataImpl.thrift.UpdateErrors) TMutation(org.apache.accumulo.core.dataImpl.thrift.TMutation) Logger(org.slf4j.Logger) Iterator(java.util.Iterator) SecurityErrorCode(org.apache.accumulo.core.clientImpl.thrift.SecurityErrorCode) Scope(io.opentelemetry.context.Scope) UTF_8(java.nio.charset.StandardCharsets.UTF_8) ClientServiceHandler(org.apache.accumulo.server.client.ClientServiceHandler) CompactionInfo(org.apache.accumulo.server.compaction.CompactionInfo) SampleNotPresentException(org.apache.accumulo.core.client.SampleNotPresentException) Constants(org.apache.accumulo.core.Constants) Authorizations(org.apache.accumulo.core.security.Authorizations) ScanBatch(org.apache.accumulo.tserver.tablet.ScanBatch) TimeUnit(java.util.concurrent.TimeUnit) Collectors.toList(java.util.stream.Collectors.toList) ThriftTableOperationException(org.apache.accumulo.core.clientImpl.thrift.ThriftTableOperationException) TabletType(org.apache.accumulo.core.clientImpl.TabletType) TDurability(org.apache.accumulo.core.tabletserver.thrift.TDurability) Tablet(org.apache.accumulo.tserver.tablet.Tablet) Cache(com.google.common.cache.Cache) Collections(java.util.Collections) Authorizations(org.apache.accumulo.core.security.Authorizations) InitialMultiScan(org.apache.accumulo.core.dataImpl.thrift.InitialMultiScan) TKeyExtent(org.apache.accumulo.core.dataImpl.thrift.TKeyExtent) TRange(org.apache.accumulo.core.dataImpl.thrift.TRange) Range(org.apache.accumulo.core.data.Range) TRowRange(org.apache.accumulo.core.dataImpl.thrift.TRowRange) ThriftSecurityException(org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException) TKeyExtent(org.apache.accumulo.core.dataImpl.thrift.TKeyExtent) KeyExtent(org.apache.accumulo.core.dataImpl.KeyExtent) MultiScanResult(org.apache.accumulo.core.dataImpl.thrift.MultiScanResult) Column(org.apache.accumulo.core.data.Column) TColumn(org.apache.accumulo.core.dataImpl.thrift.TColumn) ScanParameters(org.apache.accumulo.tserver.scan.ScanParameters) MultiScanSession(org.apache.accumulo.tserver.session.MultiScanSession) ArrayList(java.util.ArrayList) List(java.util.List) Collectors.toList(java.util.stream.Collectors.toList) NamespaceId(org.apache.accumulo.core.data.NamespaceId) HashSet(java.util.HashSet)

Example 3 with ScanParameters

use of org.apache.accumulo.tserver.scan.ScanParameters in project accumulo by apache.

the class Tablet method checkConditions.

public void checkConditions(ConditionChecker checker, Authorizations authorizations, AtomicBoolean iFlag) throws IOException {
    ScanParameters scanParams = new ScanParameters(-1, authorizations, Collections.emptySet(), null, null, false, null, -1, null);
    scanParams.setScanDispatch(ScanDispatch.builder().build());
    ScanDataSource dataSource = new ScanDataSource(this, scanParams, false, iFlag);
    try {
        SortedKeyValueIterator<Key, Value> iter = new SourceSwitchingIterator(dataSource);
        checker.check(iter);
    } catch (IOException ioe) {
        dataSource.close(true);
        throw ioe;
    } finally {
        // code in finally block because always want
        // to return mapfiles, even when exception is thrown
        dataSource.close(false);
    }
}
Also used : ScanParameters(org.apache.accumulo.tserver.scan.ScanParameters) Value(org.apache.accumulo.core.data.Value) DataFileValue(org.apache.accumulo.core.metadata.schema.DataFileValue) IOException(java.io.IOException) SourceSwitchingIterator(org.apache.accumulo.core.iteratorsImpl.system.SourceSwitchingIterator) Key(org.apache.accumulo.core.data.Key)

Aggregations

ScanParameters (org.apache.accumulo.tserver.scan.ScanParameters)3 IOException (java.io.IOException)2 HashSet (java.util.HashSet)2 TableNotFoundException (org.apache.accumulo.core.client.TableNotFoundException)2 ThriftSecurityException (org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException)2 Column (org.apache.accumulo.core.data.Column)2 Key (org.apache.accumulo.core.data.Key)2 NamespaceId (org.apache.accumulo.core.data.NamespaceId)2 Range (org.apache.accumulo.core.data.Range)2 TableId (org.apache.accumulo.core.data.TableId)2 KeyExtent (org.apache.accumulo.core.dataImpl.KeyExtent)2 InitialScan (org.apache.accumulo.core.dataImpl.thrift.InitialScan)2 MultiScanResult (org.apache.accumulo.core.dataImpl.thrift.MultiScanResult)2 ScanResult (org.apache.accumulo.core.dataImpl.thrift.ScanResult)2 TColumn (org.apache.accumulo.core.dataImpl.thrift.TColumn)2 TKeyExtent (org.apache.accumulo.core.dataImpl.thrift.TKeyExtent)2 TRange (org.apache.accumulo.core.dataImpl.thrift.TRange)2 TRowRange (org.apache.accumulo.core.dataImpl.thrift.TRowRange)2 Authorizations (org.apache.accumulo.core.security.Authorizations)2 NoSuchScanIDException (org.apache.accumulo.core.tabletserver.thrift.NoSuchScanIDException)2