Examples with AsyncConnection org.apache.hadoop.hbase.client.AsyncConnection used on opensource projects

Example 1 with AsyncConnection

use of org.apache.hadoop.hbase.client.AsyncConnection in project flink by apache.

the class HBaseRowDataAsyncLookupFunction method open.

@Override
public void open(FunctionContext context) {
    LOG.info("start open ...");
    final ExecutorService threadPool = Executors.newFixedThreadPool(THREAD_POOL_SIZE, new ExecutorThreadFactory("hbase-async-lookup-worker", Threads.LOGGING_EXCEPTION_HANDLER));
    Configuration config = prepareRuntimeConfiguration();
    CompletableFuture<AsyncConnection> asyncConnectionFuture = ConnectionFactory.createAsyncConnection(config);
    try {
        asyncConnection = asyncConnectionFuture.get();
        table = asyncConnection.getTable(TableName.valueOf(hTableName), threadPool);
        this.cache = cacheMaxSize <= 0 || cacheExpireMs <= 0 ? null : CacheBuilder.newBuilder().recordStats().expireAfterWrite(cacheExpireMs, TimeUnit.MILLISECONDS).maximumSize(cacheMaxSize).build();
        if (cache != null && context != null) {
            context.getMetricGroup().gauge("lookupCacheHitRate", (Gauge<Double>) () -> cache.stats().hitRate());
        }
    } catch (InterruptedException | ExecutionException e) {
        LOG.error("Exception while creating connection to HBase.", e);
        throw new RuntimeException("Cannot create connection to HBase.", e);
    }
    this.serde = new HBaseSerde(hbaseTableSchema, nullStringLiteral);
    LOG.info("end open.");
}

Example 2 with AsyncConnection

use of org.apache.hadoop.hbase.client.AsyncConnection in project hbase by apache.

the class AsyncClientExample method run.

@Override
public int run(String[] args) throws Exception {
    if (args.length < 1 || args.length > 2) {
        System.out.println("Usage: " + this.getClass().getName() + " tableName [num_operations]");
        return -1;
    }
    TableName tableName = TableName.valueOf(args[0]);
    int numOps = args.length > 1 ? Integer.parseInt(args[1]) : DEFAULT_NUM_OPS;
    ExecutorService threadPool = Executors.newFixedThreadPool(THREAD_POOL_SIZE, new ThreadFactoryBuilder().setNameFormat("AsyncClientExample-pool-%d").setDaemon(true).setUncaughtExceptionHandler(Threads.LOGGING_EXCEPTION_HANDLER).build());
    // We use AsyncTable here so we need to provide a separated thread pool. RawAsyncTable does not
    // need a thread pool and may have a better performance if you use it correctly as it can save
    // some context switches. But if you use RawAsyncTable incorrectly, you may have a very bad
    // impact on performance so use it with caution.
    CountDownLatch latch = new CountDownLatch(numOps);
    IntStream.range(0, numOps).forEach(i -> {
        CompletableFuture<AsyncConnection> future = getConn();
        addListener(future, (conn, error) -> {
            if (error != null) {
                LOG.warn("failed to get async connection for " + i, error);
                latch.countDown();
                return;
            }
            AsyncTable<?> table = conn.getTable(tableName, threadPool);
            addListener(table.put(new Put(getKey(i)).addColumn(FAMILY, QUAL, Bytes.toBytes(i))), (putResp, putErr) -> {
                if (putErr != null) {
                    LOG.warn("put failed for " + i, putErr);
                    latch.countDown();
                    return;
                }
                LOG.info("put for " + i + " succeeded, try getting");
                addListener(table.get(new Get(getKey(i))), (result, getErr) -> {
                    if (getErr != null) {
                        LOG.warn("get failed for " + i);
                        latch.countDown();
                        return;
                    }
                    if (result.isEmpty()) {
                        LOG.warn("get failed for " + i + ", server returns empty result");
                    } else if (!result.containsColumn(FAMILY, QUAL)) {
                        LOG.warn("get failed for " + i + ", the result does not contain " + Bytes.toString(FAMILY) + ":" + Bytes.toString(QUAL));
                    } else {
                        int v = Bytes.toInt(result.getValue(FAMILY, QUAL));
                        if (v != i) {
                            LOG.warn("get failed for " + i + ", the value of " + Bytes.toString(FAMILY) + ":" + Bytes.toString(QUAL) + " is " + v + ", exected " + i);
                        } else {
                            LOG.info("get for " + i + " succeeded");
                        }
                    }
                    latch.countDown();
                });
            });
        });
    });
    latch.await();
    closeConn().get();
    return 0;
}

Example 3 with AsyncConnection

use of org.apache.hadoop.hbase.client.AsyncConnection in project hbase by apache.

the class TestZstdDictionarySplitMerge method test.

@Test
public void test() throws Exception {
    // Create the table
    final TableName tableName = TableName.valueOf("TestZstdDictionarySplitMerge");
    final byte[] cfName = Bytes.toBytes("info");
    final String dictionaryPath = DictionaryCache.RESOURCE_SCHEME + "zstd.test.dict";
    final TableDescriptor td = TableDescriptorBuilder.newBuilder(tableName).setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(cfName).setCompressionType(Compression.Algorithm.ZSTD).setConfiguration(ZstdCodec.ZSTD_DICTIONARY_KEY, dictionaryPath).build()).build();
    final Admin admin = TEST_UTIL.getAdmin();
    admin.createTable(td, new byte[][] { Bytes.toBytes(1) });
    TEST_UTIL.waitTableAvailable(tableName);
    // Load some data
    Table t = ConnectionFactory.createConnection(conf).getTable(tableName);
    TEST_UTIL.loadNumericRows(t, cfName, 0, 100_000);
    admin.flush(tableName);
    assertTrue("Dictionary was not loaded", DictionaryCache.contains(dictionaryPath));
    TEST_UTIL.verifyNumericRows(t, cfName, 0, 100_000, 0);
    // Test split procedure
    admin.split(tableName, Bytes.toBytes(50_000));
    TEST_UTIL.waitFor(30000, new ExplainingPredicate<Exception>() {

        @Override
        public boolean evaluate() throws Exception {
            return TEST_UTIL.getMiniHBaseCluster().getRegions(tableName).size() == 3;
        }

        @Override
        public String explainFailure() throws Exception {
            return "Split has not finished yet";
        }
    });
    TEST_UTIL.waitUntilNoRegionsInTransition();
    TEST_UTIL.verifyNumericRows(t, cfName, 0, 100_000, 0);
    // Test merge procedure
    RegionInfo regionA = null;
    RegionInfo regionB = null;
    for (RegionInfo region : admin.getRegions(tableName)) {
        if (region.getStartKey().length == 0) {
            regionA = region;
        } else if (Bytes.equals(region.getStartKey(), Bytes.toBytes(1))) {
            regionB = region;
        }
    }
    assertNotNull(regionA);
    assertNotNull(regionB);
    admin.mergeRegionsAsync(new byte[][] { regionA.getRegionName(), regionB.getRegionName() }, false).get(30, TimeUnit.SECONDS);
    assertEquals(2, admin.getRegions(tableName).size());
    ServerName expected = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0).getServerName();
    assertEquals(expected, TEST_UTIL.getConnection().getRegionLocator(tableName).getRegionLocation(Bytes.toBytes(1), true).getServerName());
    try (AsyncConnection asyncConn = ConnectionFactory.createAsyncConnection(conf).get()) {
        assertEquals(expected, asyncConn.getRegionLocator(tableName).getRegionLocation(Bytes.toBytes(1), true).get().getServerName());
    }
    TEST_UTIL.verifyNumericRows(t, cfName, 0, 100_000, 0);
}

Example 4 with AsyncConnection

use of org.apache.hadoop.hbase.client.AsyncConnection in project hbase by apache.

the class PerformanceEvaluation method doLocalClients.

/*
   * Run all clients in this vm each to its own thread.
   */
static RunResult[] doLocalClients(final TestOptions opts, final Configuration conf) throws IOException, InterruptedException, ExecutionException {
    final Class<? extends TestBase> cmd = determineCommandClass(opts.cmdName);
    assert cmd != null;
    @SuppressWarnings("unchecked") Future<RunResult>[] threads = new Future[opts.numClientThreads];
    RunResult[] results = new RunResult[opts.numClientThreads];
    ExecutorService pool = Executors.newFixedThreadPool(opts.numClientThreads, new ThreadFactoryBuilder().setNameFormat("TestClient-%s").build());
    setupConnectionCount(opts);
    final Connection[] cons = new Connection[opts.connCount];
    final AsyncConnection[] asyncCons = new AsyncConnection[opts.connCount];
    for (int i = 0; i < opts.connCount; i++) {
        cons[i] = ConnectionFactory.createConnection(conf);
        asyncCons[i] = ConnectionFactory.createAsyncConnection(conf).get();
    }
    LOG.info("Created " + opts.connCount + " connections for " + opts.numClientThreads + " threads");
    for (int i = 0; i < threads.length; i++) {
        final int index = i;
        threads[i] = pool.submit(new Callable<RunResult>() {

            @Override
            public RunResult call() throws Exception {
                TestOptions threadOpts = new TestOptions(opts);
                final Connection con = cons[index % cons.length];
                final AsyncConnection asyncCon = asyncCons[index % asyncCons.length];
                if (threadOpts.startRow == 0)
                    threadOpts.startRow = index * threadOpts.perClientRunRows;
                RunResult run = runOneClient(cmd, conf, con, asyncCon, threadOpts, new Status() {

                    @Override
                    public void setStatus(final String msg) throws IOException {
                        LOG.info(msg);
                    }
                });
                LOG.info("Finished " + Thread.currentThread().getName() + " in " + run.duration + "ms over " + threadOpts.perClientRunRows + " rows");
                if (opts.latencyThreshold > 0) {
                    LOG.info("Number of replies over latency threshold " + opts.latencyThreshold + "(ms) is " + run.numbOfReplyOverThreshold);
                }
                return run;
            }
        });
    }
    pool.shutdown();
    for (int i = 0; i < threads.length; i++) {
        try {
            results[i] = threads[i].get();
        } catch (ExecutionException e) {
            throw new IOException(e.getCause());
        }
    }
    final String test = cmd.getSimpleName();
    LOG.info("[" + test + "] Summary of timings (ms): " + Arrays.toString(results));
    Arrays.sort(results);
    long total = 0;
    float avgLatency = 0;
    float avgTPS = 0;
    long replicaWins = 0;
    for (RunResult result : results) {
        total += result.duration;
        avgLatency += result.hist.getSnapshot().getMean();
        avgTPS += opts.perClientRunRows * 1.0f / result.duration;
        replicaWins += result.numOfReplyFromReplica;
    }
    // ms to second
    avgTPS *= 1000;
    avgLatency = avgLatency / results.length;
    LOG.info("[" + test + " duration ]" + "\tMin: " + results[0] + "ms" + "\tMax: " + results[results.length - 1] + "ms" + "\tAvg: " + (total / results.length) + "ms");
    LOG.info("[ Avg latency (us)]\t" + Math.round(avgLatency));
    LOG.info("[ Avg TPS/QPS]\t" + Math.round(avgTPS) + "\t row per second");
    if (opts.replicas > 1) {
        LOG.info("[results from replica regions] " + replicaWins);
    }
    for (int i = 0; i < opts.connCount; i++) {
        cons[i].close();
        asyncCons[i].close();
    }
    return results;
}

Example 5 with AsyncConnection

use of org.apache.hadoop.hbase.client.AsyncConnection in project hbase by apache.

the class CoprocClusterManager method exec.

@Override
protected Pair<Integer, String> exec(String hostname, ServiceType service, String... cmd) throws IOException {
    if (!supportedServices.contains(service)) {
        throw unsupportedServiceType(service);
    }
    // We only support actions vs. Master or Region Server processes. We're issuing those actions
    // via the coprocessor that's running within those processes. Thus, there's no support for
    // honoring the configured service user.
    final String command = StringUtils.join(cmd, " ");
    LOG.info("Executing remote command: {}, hostname:{}", command, hostname);
    try (final AsyncConnection conn = ConnectionFactory.createAsyncConnection(getConf()).join()) {
        final AsyncAdmin admin = conn.getAdmin();
        final ShellExecRequest req = ShellExecRequest.newBuilder().setCommand(command).setAwaitResponse(false).build();
        final ShellExecResponse resp;
        switch(service) {
            case HBASE_MASTER:
                // What happens if the intended action was killing a backup master? Right now we have
                // no `RestartBackupMasterAction` so it's probably fine.
                resp = masterExec(admin, req);
                break;
            case HBASE_REGIONSERVER:
                final ServerName targetHost = resolveRegionServerName(admin, hostname);
                resp = regionServerExec(admin, req, targetHost);
                break;
            default:
                throw new RuntimeException("should not happen");
        }
        if (LOG.isDebugEnabled()) {
            LOG.debug("Executed remote command: {}, exit code:{} , output:{}", command, resp.getExitCode(), resp.getStdout());
        } else {
            LOG.info("Executed remote command: {}, exit code:{}", command, resp.getExitCode());
        }
        return new Pair<>(resp.getExitCode(), resp.getStdout());
    }
}