Examples with AtomicReference java.util.concurrent.atomic.AtomicReference used on opensource projects

Example 36 with AtomicReference

use of java.util.concurrent.atomic.AtomicReference in project hbase by apache.

the class TestHCM method testConnectionClose.

private void testConnectionClose(boolean allowsInterrupt) throws Exception {
    TableName tableName = TableName.valueOf("HCM-testConnectionClose" + allowsInterrupt);
    TEST_UTIL.createTable(tableName, FAM_NAM).close();
    boolean previousBalance = TEST_UTIL.getAdmin().setBalancerRunning(false, true);
    Configuration c2 = new Configuration(TEST_UTIL.getConfiguration());
    // We want to work on a separate connection.
    c2.set(HConstants.HBASE_CLIENT_INSTANCE_ID, String.valueOf(-1));
    // retry a lot
    c2.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 100);
    // don't wait between retries.
    c2.setInt(HConstants.HBASE_CLIENT_PAUSE, 1);
    // Server do not really expire
    c2.setInt(RpcClient.FAILED_SERVER_EXPIRY_KEY, 0);
    c2.setBoolean(RpcClient.SPECIFIC_WRITE_THREAD, allowsInterrupt);
    // to avoid the client to be stuck when do the Get
    c2.setInt(HConstants.HBASE_CLIENT_META_OPERATION_TIMEOUT, 10000);
    c2.setInt(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT, 10000);
    c2.setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, 5000);
    Connection connection = ConnectionFactory.createConnection(c2);
    final Table table = connection.getTable(tableName);
    Put put = new Put(ROW);
    put.addColumn(FAM_NAM, ROW, ROW);
    table.put(put);
    // 4 steps: ready=0; doGets=1; mustStop=2; stopped=3
    final AtomicInteger step = new AtomicInteger(0);
    final AtomicReference<Throwable> failed = new AtomicReference<>(null);
    Thread t = new Thread("testConnectionCloseThread") {

        @Override
        public void run() {
            int done = 0;
            try {
                step.set(1);
                while (step.get() == 1) {
                    Get get = new Get(ROW);
                    table.get(get);
                    done++;
                    if (done % 100 == 0)
                        LOG.info("done=" + done);
                    // without the sleep, will cause the exception for too many files in
                    // org.apache.hadoop.hdfs.server.datanode.DataXceiver
                    Thread.sleep(100);
                }
            } catch (Throwable t) {
                failed.set(t);
                LOG.error(t);
            }
            step.set(3);
        }
    };
    t.start();
    TEST_UTIL.waitFor(20000, new Waiter.Predicate<Exception>() {

        @Override
        public boolean evaluate() throws Exception {
            return step.get() == 1;
        }
    });
    ServerName sn;
    try (RegionLocator rl = connection.getRegionLocator(tableName)) {
        sn = rl.getRegionLocation(ROW).getServerName();
    }
    ConnectionImplementation conn = (ConnectionImplementation) connection;
    RpcClient rpcClient = conn.getRpcClient();
    LOG.info("Going to cancel connections. connection=" + conn.toString() + ", sn=" + sn);
    for (int i = 0; i < 5000; i++) {
        rpcClient.cancelConnections(sn);
        Thread.sleep(5);
    }
    step.compareAndSet(1, 2);
    // The test may fail here if the thread doing the gets is stuck. The way to find
    //  out what's happening is to look for the thread named 'testConnectionCloseThread'
    TEST_UTIL.waitFor(40000, new Waiter.Predicate<Exception>() {

        @Override
        public boolean evaluate() throws Exception {
            return step.get() == 3;
        }
    });
    table.close();
    connection.close();
    Assert.assertTrue("Unexpected exception is " + failed.get(), failed.get() == null);
    TEST_UTIL.getAdmin().setBalancerRunning(previousBalance, true);
}

Example 37 with AtomicReference

use of java.util.concurrent.atomic.AtomicReference in project hive by apache.

the class TezSessionPoolManager method startPool.

public void startPool() throws Exception {
    if (initialSessions.isEmpty())
        return;
    int threadCount = Math.min(initialSessions.size(), HiveConf.getIntVar(initConf, ConfVars.HIVE_SERVER2_TEZ_SESSION_MAX_INIT_THREADS));
    Preconditions.checkArgument(threadCount > 0);
    if (threadCount == 1) {
        while (true) {
            TezSessionPoolSession session = initialSessions.poll();
            if (session == null)
                break;
            startInitialSession(session);
        }
    } else {
        // TODO What is this doing now ?
        final SessionState parentSessionState = SessionState.get();
        // The runnable has no mutable state, so each thread can run the same thing.
        final AtomicReference<Exception> firstError = new AtomicReference<>(null);
        Runnable runnable = new Runnable() {

            public void run() {
                if (parentSessionState != null) {
                    SessionState.setCurrentSessionState(parentSessionState);
                }
                while (true) {
                    TezSessionPoolSession session = initialSessions.poll();
                    if (session == null)
                        break;
                    try {
                        startInitialSession(session);
                    } catch (Exception e) {
                        if (!firstError.compareAndSet(null, e)) {
                            LOG.error("Failed to start session; ignoring due to previous error", e);
                        // TODO Why even continue after this. We're already in a state where things are messed up ?
                        }
                    }
                }
            }
        };
        Thread[] threads = new Thread[threadCount - 1];
        for (int i = 0; i < threads.length; ++i) {
            threads[i] = new Thread(runnable, "Tez session init " + i);
            threads[i].start();
        }
        runnable.run();
        for (int i = 0; i < threads.length; ++i) {
            threads[i].join();
        }
        Exception ex = firstError.get();
        if (ex != null) {
            throw ex;
        }
    }
    if (expirationThread != null) {
        expirationThread.start();
        restartThread.start();
    }
}

Example 38 with AtomicReference

use of java.util.concurrent.atomic.AtomicReference in project hbase by apache.

the class TestHRegion method testBatchPut_whileMultipleRowLocksHeld.

@Test
public void testBatchPut_whileMultipleRowLocksHeld() throws Exception {
    byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
    byte[] qual = Bytes.toBytes("qual");
    byte[] val = Bytes.toBytes("val");
    this.region = initHRegion(tableName, method, CONF, cf);
    MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
    try {
        long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
        metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
        final Put[] puts = new Put[10];
        for (int i = 0; i < 10; i++) {
            puts[i] = new Put(Bytes.toBytes("row_" + i));
            puts[i].addColumn(cf, qual, val);
        }
        puts[5].addColumn(Bytes.toBytes("BAD_CF"), qual, val);
        LOG.info("batchPut will have to break into four batches to avoid row locks");
        RowLock rowLock1 = region.getRowLock(Bytes.toBytes("row_2"));
        RowLock rowLock2 = region.getRowLock(Bytes.toBytes("row_1"));
        RowLock rowLock3 = region.getRowLock(Bytes.toBytes("row_3"));
        RowLock rowLock4 = region.getRowLock(Bytes.toBytes("row_3"), true);
        MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(CONF);
        final AtomicReference<OperationStatus[]> retFromThread = new AtomicReference<>();
        final CountDownLatch startingPuts = new CountDownLatch(1);
        final CountDownLatch startingClose = new CountDownLatch(1);
        TestThread putter = new TestThread(ctx) {

            @Override
            public void doWork() throws IOException {
                startingPuts.countDown();
                retFromThread.set(region.batchMutate(puts));
            }
        };
        LOG.info("...starting put thread while holding locks");
        ctx.addThread(putter);
        ctx.startThreads();
        // Now attempt to close the region from another thread.  Prior to HBASE-12565
        // this would cause the in-progress batchMutate operation to to fail with
        // exception because it use to release and re-acquire the close-guard lock
        // between batches.  Caller then didn't get status indicating which writes succeeded.
        // We now expect this thread to block until the batchMutate call finishes.
        Thread regionCloseThread = new TestThread(ctx) {

            @Override
            public void doWork() {
                try {
                    startingPuts.await();
                    // Give some time for the batch mutate to get in.
                    // We don't want to race with the mutate
                    Thread.sleep(10);
                    startingClose.countDown();
                    HBaseTestingUtility.closeRegionAndWAL(region);
                } catch (IOException e) {
                    throw new RuntimeException(e);
                } catch (InterruptedException e) {
                    throw new RuntimeException(e);
                }
            }
        };
        regionCloseThread.start();
        startingClose.await();
        startingPuts.await();
        Thread.sleep(100);
        LOG.info("...releasing row lock 1, which should let put thread continue");
        rowLock1.release();
        rowLock2.release();
        rowLock3.release();
        waitForCounter(source, "syncTimeNumOps", syncs + 1);
        LOG.info("...joining on put thread");
        ctx.stop();
        regionCloseThread.join();
        OperationStatus[] codes = retFromThread.get();
        for (int i = 0; i < codes.length; i++) {
            assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS, codes[i].getOperationStatusCode());
        }
        rowLock4.release();
    } finally {
        HBaseTestingUtility.closeRegionAndWAL(this.region);
        this.region = null;
    }
}

Example 39 with AtomicReference

use of java.util.concurrent.atomic.AtomicReference in project hbase by apache.

the class TestRegionReplicaFailover method testSecondaryRegionKillWhilePrimaryIsAcceptingWrites.

/**
   * Tests the case where there are 3 region replicas and the primary is continuously accepting
   * new writes while one of the secondaries is killed. Verification is done for both of the
   * secondary replicas.
   */
@Test(timeout = 120000)
public void testSecondaryRegionKillWhilePrimaryIsAcceptingWrites() throws Exception {
    try (Connection connection = ConnectionFactory.createConnection(HTU.getConfiguration());
        Table table = connection.getTable(htd.getTableName());
        Admin admin = connection.getAdmin()) {
        // start a thread to do the loading of primary
        // start with some base
        HTU.loadNumericRows(table, fam, 0, 1000);
        admin.flush(table.getName());
        HTU.loadNumericRows(table, fam, 1000, 2000);
        final AtomicReference<Throwable> ex = new AtomicReference<>(null);
        final AtomicBoolean done = new AtomicBoolean(false);
        final AtomicInteger key = new AtomicInteger(2000);
        Thread loader = new Thread() {

            @Override
            public void run() {
                while (!done.get()) {
                    try {
                        HTU.loadNumericRows(table, fam, key.get(), key.get() + 1000);
                        key.addAndGet(1000);
                    } catch (Throwable e) {
                        ex.compareAndSet(null, e);
                    }
                }
            }
        };
        loader.start();
        Thread aborter = new Thread() {

            @Override
            public void run() {
                try {
                    boolean aborted = false;
                    for (RegionServerThread rs : HTU.getMiniHBaseCluster().getRegionServerThreads()) {
                        for (Region r : rs.getRegionServer().getOnlineRegions(htd.getTableName())) {
                            if (r.getRegionInfo().getReplicaId() == 1) {
                                LOG.info("Aborting region server hosting secondary region replica");
                                rs.getRegionServer().abort("for test");
                                aborted = true;
                            }
                        }
                    }
                    assertTrue(aborted);
                } catch (Throwable e) {
                    ex.compareAndSet(null, e);
                }
            }

            ;
        };
        aborter.start();
        aborter.join();
        done.set(true);
        loader.join();
        assertNull(ex.get());
        // assert that the test is working as designed
        assertTrue(key.get() > 1000);
        LOG.info("Loaded up to key :" + key.get());
        verifyNumericRowsWithTimeout(table, fam, 0, key.get(), 0, 30000);
        verifyNumericRowsWithTimeout(table, fam, 0, key.get(), 1, 30000);
        verifyNumericRowsWithTimeout(table, fam, 0, key.get(), 2, 30000);
    }
    // restart the region server
    HTU.getMiniHBaseCluster().startRegionServer();
}

Example 40 with AtomicReference

use of java.util.concurrent.atomic.AtomicReference in project hbase by apache.

the class TestRegionReplicas method testFlushAndCompactionsInPrimary.

@Test(timeout = 300000)
public void testFlushAndCompactionsInPrimary() throws Exception {
    long runtime = 30 * 1000;
    // enable store file refreshing
    // 100ms refresh is a lot
    final int refreshPeriod = 100;
    HTU.getConfiguration().setInt("hbase.hstore.compactionThreshold", 3);
    HTU.getConfiguration().setInt(StorefileRefresherChore.REGIONSERVER_STOREFILE_REFRESH_PERIOD, refreshPeriod);
    // restart the region server so that it starts the refresher chore
    restartRegionServer();
    final int startKey = 0, endKey = 1000;
    try {
        openRegion(HTU, getRS(), hriSecondary);
        //load some data to primary so that reader won't fail
        HTU.loadNumericRows(table, f, startKey, endKey);
        TestRegionServerNoMaster.flushRegion(HTU, hriPrimary);
        // ensure that chore is run
        Threads.sleep(2 * refreshPeriod);
        final AtomicBoolean running = new AtomicBoolean(true);
        @SuppressWarnings("unchecked") final AtomicReference<Exception>[] exceptions = new AtomicReference[3];
        for (int i = 0; i < exceptions.length; i++) {
            exceptions[i] = new AtomicReference<>();
        }
        Runnable writer = new Runnable() {

            int key = startKey;

            @Override
            public void run() {
                try {
                    while (running.get()) {
                        byte[] data = Bytes.toBytes(String.valueOf(key));
                        Put put = new Put(data);
                        put.addColumn(f, null, data);
                        table.put(put);
                        key++;
                        if (key == endKey)
                            key = startKey;
                    }
                } catch (Exception ex) {
                    LOG.warn(ex);
                    exceptions[0].compareAndSet(null, ex);
                }
            }
        };
        Runnable flusherCompactor = new Runnable() {

            Random random = new Random();

            @Override
            public void run() {
                try {
                    while (running.get()) {
                        // flush or compact
                        if (random.nextBoolean()) {
                            TestRegionServerNoMaster.flushRegion(HTU, hriPrimary);
                        } else {
                            HTU.compact(table.getName(), random.nextBoolean());
                        }
                    }
                } catch (Exception ex) {
                    LOG.warn(ex);
                    exceptions[1].compareAndSet(null, ex);
                }
            }
        };
        Runnable reader = new Runnable() {

            Random random = new Random();

            @Override
            public void run() {
                try {
                    while (running.get()) {
                        // whether to do a close and open
                        if (random.nextInt(10) == 0) {
                            try {
                                closeRegion(HTU, getRS(), hriSecondary);
                            } catch (Exception ex) {
                                LOG.warn("Failed closing the region " + hriSecondary + " " + StringUtils.stringifyException(ex));
                                exceptions[2].compareAndSet(null, ex);
                            }
                            try {
                                openRegion(HTU, getRS(), hriSecondary);
                            } catch (Exception ex) {
                                LOG.warn("Failed opening the region " + hriSecondary + " " + StringUtils.stringifyException(ex));
                                exceptions[2].compareAndSet(null, ex);
                            }
                        }
                        int key = random.nextInt(endKey - startKey) + startKey;
                        assertGetRpc(hriSecondary, key, true);
                    }
                } catch (Exception ex) {
                    LOG.warn("Failed getting the value in the region " + hriSecondary + " " + StringUtils.stringifyException(ex));
                    exceptions[2].compareAndSet(null, ex);
                }
            }
        };
        LOG.info("Starting writer and reader");
        ExecutorService executor = Executors.newFixedThreadPool(3);
        executor.submit(writer);
        executor.submit(flusherCompactor);
        executor.submit(reader);
        // wait for threads
        Threads.sleep(runtime);
        running.set(false);
        executor.shutdown();
        executor.awaitTermination(30, TimeUnit.SECONDS);
        for (AtomicReference<Exception> exRef : exceptions) {
            Assert.assertNull(exRef.get());
        }
    } finally {
        HTU.deleteNumericRows(table, HConstants.CATALOG_FAMILY, startKey, endKey);
        closeRegion(HTU, getRS(), hriSecondary);
    }
}