Search in sources :

Example 16 with MutableBoolean

use of org.apache.commons.lang.mutable.MutableBoolean in project midpoint by Evolveum.

the class SecurityEnforcerImpl method processAuthorizationContainerDelta.

private <C extends Containerable> boolean processAuthorizationContainerDelta(ContainerDelta<C> cdelta, final Collection<ItemPath> allowedItems) {
    final MutableBoolean itemDecision = new MutableBoolean(true);
    cdelta.foreach(cval -> {
        if (!isContainerAllowed(cval, allowedItems)) {
            itemDecision.setValue(false);
        }
    });
    return itemDecision.booleanValue();
}
Also used : MutableBoolean(org.apache.commons.lang.mutable.MutableBoolean)

Example 17 with MutableBoolean

use of org.apache.commons.lang.mutable.MutableBoolean in project hbase by apache.

the class ClientScanner method loadCache.

/**
   * Contact the servers to load more {@link Result}s in the cache.
   */
protected void loadCache() throws IOException {
    // check if scanner was closed during previous prefetch
    if (closed) {
        return;
    }
    long remainingResultSize = maxScannerResultSize;
    int countdown = this.caching;
    // This is possible if we just stopped at the boundary of a region in the previous call.
    if (callable == null) {
        if (!moveToNextRegion()) {
            return;
        }
    }
    // This flag is set when we want to skip the result returned. We do
    // this when we reset scanner because it split under us.
    MutableBoolean retryAfterOutOfOrderException = new MutableBoolean(true);
    // Even if we are retrying due to UnknownScannerException, ScannerResetException, etc. we should
    // make sure that we are not retrying indefinitely.
    int retriesLeft = getRetries();
    for (; ; ) {
        Result[] values;
        try {
            // Server returns a null values if scanning is to stop. Else,
            // returns an empty array if scanning is to go on and we've just
            // exhausted current region.
            // now we will also fetch data when openScanner, so do not make a next call again if values
            // is already non-null.
            values = call(callable, caller, scannerTimeout, true);
            // of the loop as it happens for the cases where we see exceptions.
            if (callable.switchedToADifferentReplica()) {
                // Any accumulated partial results are no longer valid since the callable will
                // openScanner with the correct startkey and we must pick up from there
                scanResultCache.clear();
                this.currentRegion = callable.getHRegionInfo();
            }
            retryAfterOutOfOrderException.setValue(true);
        } catch (DoNotRetryIOException e) {
            handleScanError(e, retryAfterOutOfOrderException, retriesLeft--);
            // reopen the scanner
            if (!moveToNextRegion()) {
                break;
            }
            continue;
        }
        long currentTime = System.currentTimeMillis();
        if (this.scanMetrics != null) {
            this.scanMetrics.sumOfMillisSecBetweenNexts.addAndGet(currentTime - lastNext);
        }
        lastNext = currentTime;
        // Groom the array of Results that we received back from the server before adding that
        // Results to the scanner's cache. If partial results are not allowed to be seen by the
        // caller, all book keeping will be performed within this method.
        Result[] resultsToAddToCache = scanResultCache.addAndGet(values, callable.isHeartbeatMessage());
        if (resultsToAddToCache.length > 0) {
            for (Result rs : resultsToAddToCache) {
                cache.add(rs);
                long estimatedHeapSizeOfResult = calcEstimatedSize(rs);
                countdown--;
                remainingResultSize -= estimatedHeapSizeOfResult;
                addEstimatedSize(estimatedHeapSizeOfResult);
                this.lastResult = rs;
            }
            if (scan.getLimit() > 0) {
                int newLimit = scan.getLimit() - numberOfIndividualRows(Arrays.asList(resultsToAddToCache));
                assert newLimit >= 0;
                scan.setLimit(newLimit);
            }
        }
        if (scanExhausted(values)) {
            closeScanner();
            closed = true;
            break;
        }
        boolean regionExhausted = regionExhausted(values);
        if (callable.isHeartbeatMessage()) {
            if (!cache.isEmpty()) {
                // unnecesary delays to the caller
                if (LOG.isTraceEnabled()) {
                    LOG.trace("Heartbeat message received and cache contains Results." + " Breaking out of scan loop");
                }
                // closeScannerIfExhausted
                break;
            }
        }
        if (countdown <= 0) {
            // we have enough result.
            closeScannerIfExhausted(regionExhausted);
            break;
        }
        if (remainingResultSize <= 0) {
            if (!cache.isEmpty()) {
                closeScannerIfExhausted(regionExhausted);
                break;
            } else {
                // we have reached the max result size but we still can not find anything to return to the
                // user. Reset the maxResultSize and try again.
                remainingResultSize = maxScannerResultSize;
            }
        }
        // we are done with the current region
        if (regionExhausted) {
            if (!moveToNextRegion()) {
                break;
            }
        }
    }
}
Also used : DoNotRetryIOException(org.apache.hadoop.hbase.DoNotRetryIOException) MutableBoolean(org.apache.commons.lang.mutable.MutableBoolean)

Example 18 with MutableBoolean

use of org.apache.commons.lang.mutable.MutableBoolean in project pinot by linkedin.

the class RoutingTableTest method testTimeBoundaryRegression.

@Test
public void testTimeBoundaryRegression() throws Exception {
    final FakePropertyStore propertyStore = new FakePropertyStore();
    final OfflineSegmentZKMetadata offlineSegmentZKMetadata = new OfflineSegmentZKMetadata();
    offlineSegmentZKMetadata.setTimeUnit(TimeUnit.DAYS);
    offlineSegmentZKMetadata.setEndTime(1234L);
    propertyStore.setContents(ZKMetadataProvider.constructPropertyStorePathForSegment("myTable_OFFLINE", "someSegment_0"), offlineSegmentZKMetadata.toZNRecord());
    final ExternalView offlineExternalView = new ExternalView("myTable_OFFLINE");
    offlineExternalView.setState("someSegment_0", "Server_1.2.3.4_1234", "ONLINE");
    final MutableBoolean timeBoundaryUpdated = new MutableBoolean(false);
    HelixExternalViewBasedRouting routingTable = new HelixExternalViewBasedRouting(propertyStore, NO_LLC_ROUTING, null, new BaseConfiguration()) {

        @Override
        protected ExternalView fetchExternalView(String table) {
            return offlineExternalView;
        }

        @Override
        protected void updateTimeBoundary(String tableName, ExternalView externalView) {
            if (tableName.equals("myTable_OFFLINE")) {
                timeBoundaryUpdated.setValue(true);
            }
        }
    };
    routingTable.setBrokerMetrics(new BrokerMetrics(new MetricsRegistry()));
    Assert.assertFalse(timeBoundaryUpdated.booleanValue());
    final ArrayList<InstanceConfig> instanceConfigList = new ArrayList<>();
    instanceConfigList.add(new InstanceConfig("Server_1.2.3.4_1234"));
    routingTable.markDataResourceOnline("myTable_OFFLINE", offlineExternalView, instanceConfigList);
    routingTable.markDataResourceOnline("myTable_REALTIME", new ExternalView("myTable_REALTIME"), null);
    Assert.assertTrue(timeBoundaryUpdated.booleanValue());
}
Also used : ExternalView(org.apache.helix.model.ExternalView) BaseConfiguration(org.apache.commons.configuration.BaseConfiguration) MetricsRegistry(com.yammer.metrics.core.MetricsRegistry) InstanceConfig(org.apache.helix.model.InstanceConfig) OfflineSegmentZKMetadata(com.linkedin.pinot.common.metadata.segment.OfflineSegmentZKMetadata) MutableBoolean(org.apache.commons.lang.mutable.MutableBoolean) ArrayList(java.util.ArrayList) BrokerMetrics(com.linkedin.pinot.common.metrics.BrokerMetrics) Test(org.testng.annotations.Test)

Example 19 with MutableBoolean

use of org.apache.commons.lang.mutable.MutableBoolean in project hadoop by apache.

the class TestDelegationTokenAuthenticationHandlerWithMocks method testWriterNotClosed.

@Test
public void testWriterNotClosed() throws Exception {
    Properties conf = new Properties();
    conf.put(KerberosDelegationTokenAuthenticationHandler.TOKEN_KIND, "foo");
    conf.put(DelegationTokenAuthenticationHandler.JSON_MAPPER_PREFIX + "AUTO_CLOSE_TARGET", "false");
    DelegationTokenAuthenticationHandler noAuthCloseHandler = new MockDelegationTokenAuthenticationHandler();
    try {
        noAuthCloseHandler.initTokenManager(conf);
        noAuthCloseHandler.initJsonFactory(conf);
        DelegationTokenAuthenticator.DelegationTokenOperation op = GETDELEGATIONTOKEN;
        HttpServletRequest request = Mockito.mock(HttpServletRequest.class);
        HttpServletResponse response = Mockito.mock(HttpServletResponse.class);
        Mockito.when(request.getQueryString()).thenReturn(DelegationTokenAuthenticator.OP_PARAM + "=" + op.toString());
        Mockito.when(request.getMethod()).thenReturn(op.getHttpMethod());
        AuthenticationToken token = Mockito.mock(AuthenticationToken.class);
        Mockito.when(token.getUserName()).thenReturn("user");
        final MutableBoolean closed = new MutableBoolean();
        PrintWriter printWriterCloseCount = new PrintWriter(new StringWriter()) {

            @Override
            public void close() {
                closed.setValue(true);
                super.close();
            }

            @Override
            public void write(String str) {
                if (closed.booleanValue()) {
                    throw new RuntimeException("already closed!");
                }
                super.write(str);
            }
        };
        Mockito.when(response.getWriter()).thenReturn(printWriterCloseCount);
        Assert.assertFalse(noAuthCloseHandler.managementOperation(token, request, response));
    } finally {
        noAuthCloseHandler.destroy();
    }
}
Also used : AuthenticationToken(org.apache.hadoop.security.authentication.server.AuthenticationToken) MutableBoolean(org.apache.commons.lang.mutable.MutableBoolean) HttpServletResponse(javax.servlet.http.HttpServletResponse) Properties(java.util.Properties) HttpServletRequest(javax.servlet.http.HttpServletRequest) StringWriter(java.io.StringWriter) DelegationTokenOperation(org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticator.DelegationTokenOperation) PrintWriter(java.io.PrintWriter) Test(org.junit.Test)

Example 20 with MutableBoolean

use of org.apache.commons.lang.mutable.MutableBoolean in project hadoop by apache.

the class TestEnhancedByteBufferAccess method testZeroCopyMmapCache.

@Test
public void testZeroCopyMmapCache() throws Exception {
    HdfsConfiguration conf = initZeroCopyTest();
    MiniDFSCluster cluster = null;
    final Path TEST_PATH = new Path("/a");
    final int TEST_FILE_LENGTH = 5 * BLOCK_SIZE;
    final int RANDOM_SEED = 23453;
    final String CONTEXT = "testZeroCopyMmapCacheContext";
    FSDataInputStream fsIn = null;
    ByteBuffer[] results = { null, null, null, null };
    DistributedFileSystem fs = null;
    conf.set(HdfsClientConfigKeys.DFS_CLIENT_CONTEXT, CONTEXT);
    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
    cluster.waitActive();
    fs = cluster.getFileSystem();
    DFSTestUtil.createFile(fs, TEST_PATH, TEST_FILE_LENGTH, (short) 1, RANDOM_SEED);
    try {
        DFSTestUtil.waitReplication(fs, TEST_PATH, (short) 1);
    } catch (InterruptedException e) {
        Assert.fail("unexpected InterruptedException during " + "waitReplication: " + e);
    } catch (TimeoutException e) {
        Assert.fail("unexpected TimeoutException during " + "waitReplication: " + e);
    }
    fsIn = fs.open(TEST_PATH);
    byte[] original = new byte[TEST_FILE_LENGTH];
    IOUtils.readFully(fsIn, original, 0, TEST_FILE_LENGTH);
    fsIn.close();
    fsIn = fs.open(TEST_PATH);
    final ShortCircuitCache cache = ClientContext.get(CONTEXT, conf).getShortCircuitCache();
    cache.accept(new CountingVisitor(0, 5, 5, 0));
    results[0] = fsIn.read(null, BLOCK_SIZE, EnumSet.of(ReadOption.SKIP_CHECKSUMS));
    fsIn.seek(0);
    results[1] = fsIn.read(null, BLOCK_SIZE, EnumSet.of(ReadOption.SKIP_CHECKSUMS));
    // The mmap should be of the first block of the file.
    final ExtendedBlock firstBlock = DFSTestUtil.getFirstBlock(fs, TEST_PATH);
    cache.accept(new CacheVisitor() {

        @Override
        public void visit(int numOutstandingMmaps, Map<ExtendedBlockId, ShortCircuitReplica> replicas, Map<ExtendedBlockId, InvalidToken> failedLoads, LinkedMap evictable, LinkedMap evictableMmapped) {
            ShortCircuitReplica replica = replicas.get(new ExtendedBlockId(firstBlock.getBlockId(), firstBlock.getBlockPoolId()));
            Assert.assertNotNull(replica);
            Assert.assertTrue(replica.hasMmap());
            // The replica should not yet be evictable, since we have it open.
            Assert.assertNull(replica.getEvictableTimeNs());
        }
    });
    // Read more blocks.
    results[2] = fsIn.read(null, BLOCK_SIZE, EnumSet.of(ReadOption.SKIP_CHECKSUMS));
    results[3] = fsIn.read(null, BLOCK_SIZE, EnumSet.of(ReadOption.SKIP_CHECKSUMS));
    // we should have 3 mmaps, 1 evictable
    cache.accept(new CountingVisitor(3, 5, 2, 0));
    // using a very quick timeout)
    for (ByteBuffer buffer : results) {
        if (buffer != null) {
            fsIn.releaseBuffer(buffer);
        }
    }
    fsIn.close();
    GenericTestUtils.waitFor(new Supplier<Boolean>() {

        public Boolean get() {
            final MutableBoolean finished = new MutableBoolean(false);
            cache.accept(new CacheVisitor() {

                @Override
                public void visit(int numOutstandingMmaps, Map<ExtendedBlockId, ShortCircuitReplica> replicas, Map<ExtendedBlockId, InvalidToken> failedLoads, LinkedMap evictable, LinkedMap evictableMmapped) {
                    finished.setValue(evictableMmapped.isEmpty());
                }
            });
            return finished.booleanValue();
        }
    }, 10, 60000);
    cache.accept(new CountingVisitor(0, -1, -1, -1));
    fs.close();
    cluster.shutdown();
}
Also used : ExtendedBlockId(org.apache.hadoop.hdfs.ExtendedBlockId) LinkedMap(org.apache.commons.collections.map.LinkedMap) CacheVisitor(org.apache.hadoop.hdfs.shortcircuit.ShortCircuitCache.CacheVisitor) MutableBoolean(org.apache.commons.lang.mutable.MutableBoolean) TimeoutException(java.util.concurrent.TimeoutException) MiniDFSCluster(org.apache.hadoop.hdfs.MiniDFSCluster) MutableBoolean(org.apache.commons.lang.mutable.MutableBoolean) ExtendedBlock(org.apache.hadoop.hdfs.protocol.ExtendedBlock) HdfsConfiguration(org.apache.hadoop.hdfs.HdfsConfiguration) DistributedFileSystem(org.apache.hadoop.hdfs.DistributedFileSystem) ShortCircuitCache(org.apache.hadoop.hdfs.shortcircuit.ShortCircuitCache) ByteBuffer(java.nio.ByteBuffer) ShortCircuitReplica(org.apache.hadoop.hdfs.shortcircuit.ShortCircuitReplica) InvalidToken(org.apache.hadoop.security.token.SecretManager.InvalidToken) Map(java.util.Map) LinkedMap(org.apache.commons.collections.map.LinkedMap) Test(org.junit.Test)

Aggregations

MutableBoolean (org.apache.commons.lang.mutable.MutableBoolean)27 Test (org.junit.Test)13 ArrayList (java.util.ArrayList)6 IOException (java.io.IOException)5 Map (java.util.Map)5 ExtendedBlockId (org.apache.hadoop.hdfs.ExtendedBlockId)5 Versioned (voldemort.versioning.Versioned)5 HashMap (java.util.HashMap)4 CacheVisitor (org.apache.hadoop.hdfs.shortcircuit.ShortCircuitCache.CacheVisitor)4 VectorClock (voldemort.versioning.VectorClock)4 Slot (org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm.Slot)3 com.evolveum.midpoint.prism (com.evolveum.midpoint.prism)2 ItemPath (com.evolveum.midpoint.prism.path.ItemPath)2 PolyString (com.evolveum.midpoint.prism.polystring.PolyString)2 Source (com.evolveum.midpoint.repo.common.expression.Source)2 ObjectType (com.evolveum.midpoint.xml.ns._public.common.common_3.ObjectType)2 LinkedMap (org.apache.commons.collections.map.LinkedMap)2 Validate (org.apache.commons.lang.Validate)2 Configuration (org.apache.hadoop.conf.Configuration)2 DistributedFileSystem (org.apache.hadoop.hdfs.DistributedFileSystem)2