Examples with RowKeyDistributorByHashPrefix io.cdap.cdap.hbase.wd.RowKeyDistributorByHashPrefix used on opensource projects

Example 1 with RowKeyDistributorByHashPrefix

use of io.cdap.cdap.hbase.wd.RowKeyDistributorByHashPrefix in project cdap by caskdata.

the class HBaseMetricsTable method initializeVars.

private void initializeVars(CConfiguration cConf, DatasetSpecification spec) {
    this.scanExecutor = null;
    this.rowKeyDistributor = null;
    RejectedExecutionHandler callerRunsPolicy = (r, executor) -> {
        REJECTION_LOG.info("No more threads in the HBase scan thread pool. Consider increase {}. Performing scan in caller thread {}", Constants.Metrics.METRICS_HBASE_MAX_SCAN_THREADS, Thread.currentThread().getName());
        // Runs it from the caller thread
        if (!executor.isShutdown()) {
            r.run();
        }
    };
    int maxScanThread = cConf.getInt(Constants.Metrics.METRICS_HBASE_MAX_SCAN_THREADS);
    // Creates a executor that will shrink to 0 threads if left idle
    // Uses daemon thread, hence no need to worry about shutdown
    // When all threads are busy, use the caller thread to execute
    this.scanExecutor = new ThreadPoolExecutor(0, maxScanThread, 60L, TimeUnit.SECONDS, new SynchronousQueue<Runnable>(), Threads.createDaemonThreadFactory("metrics-hbase-scanner-%d"), callerRunsPolicy);
    this.rowKeyDistributor = new RowKeyDistributorByHashPrefix(new RowKeyDistributorByHashPrefix.OneByteSimpleHash(spec.getIntProperty(Constants.Metrics.METRICS_HBASE_TABLE_SPLITS, 16)));
}

Example 2 with RowKeyDistributorByHashPrefix

use of io.cdap.cdap.hbase.wd.RowKeyDistributorByHashPrefix in project cdap by caskdata.

the class HBaseTableFactory method createTable.

/**
 * Creates a new instance of {@link Table} for the given {@link TableId}. If the hbase table doesn't
 * exist, a new one will be created with the given number of splits.
 */
private HTableWithRowKeyDistributor createTable(TableId tableId, int splits, Class<? extends Coprocessor> coprocessor) throws IOException {
    // Lookup the table descriptor from the cache first. If it is there, we assume the HBase table exists
    // Otherwise, attempt to create it.
    Table table = null;
    HTableDescriptor htd = tableDescriptors.get(tableId);
    if (htd == null) {
        synchronized (this) {
            htd = tableDescriptors.get(tableId);
            if (htd == null) {
                boolean tableExists;
                try (HBaseAdmin admin = new HBaseAdmin(hConf)) {
                    tableExists = tableUtil.tableExists(admin, tableId);
                }
                // Create the table if the table doesn't exist
                try (HBaseDDLExecutor ddlExecutor = ddlExecutorFactory.get()) {
                    // If table exists, then skip creating coprocessor etc
                    if (!tableExists) {
                        TableId metadataTableId = tableUtil.createHTableId(NamespaceId.SYSTEM, cConf.get(Constants.MessagingSystem.METADATA_TABLE_NAME));
                        ColumnFamilyDescriptorBuilder cfdBuilder = HBaseTableUtil.getColumnFamilyDescriptorBuilder(Bytes.toString(COLUMN_FAMILY), hConf);
                        TableDescriptorBuilder tdBuilder = HBaseTableUtil.getTableDescriptorBuilder(tableId, cConf).addColumnFamily(cfdBuilder.build()).addProperty(Constants.MessagingSystem.HBASE_MESSAGING_TABLE_PREFIX_NUM_BYTES, Integer.toString(1)).addProperty(Constants.MessagingSystem.KEY_DISTRIBUTOR_BUCKETS_ATTR, Integer.toString(splits)).addProperty(Constants.MessagingSystem.HBASE_METADATA_TABLE_NAMESPACE, metadataTableId.getNamespace()).addProperty(HTableDescriptor.SPLIT_POLICY, cConf.get(Constants.MessagingSystem.TABLE_HBASE_SPLIT_POLICY)).addCoprocessor(coprocessorManager.getCoprocessorDescriptor(coprocessor, Coprocessor.PRIORITY_USER));
                        // Set the key distributor size the same as the initial number of splits,
                        // essentially one bucket per split.
                        byte[][] splitKeys = HBaseTableUtil.getSplitKeys(splits, splits, new RowKeyDistributorByHashPrefix(new OneByteSimpleHash(splits)));
                        ddlExecutor.createTableIfNotExists(tdBuilder.build(), splitKeys);
                        table = tableUtil.createTable(hConf, tableId);
                        htd = table.getTableDescriptor();
                        tableDescriptors.put(tableId, htd);
                    } else {
                        table = tableUtil.createTable(hConf, tableId);
                        htd = table.getTableDescriptor();
                        tableDescriptors.put(tableId, htd);
                    }
                }
            }
        }
    }
    if (table == null) {
        table = tableUtil.createTable(hConf, tableId);
    }
    return new HTableWithRowKeyDistributor(table, new RowKeyDistributorByHashPrefix(new OneByteSimpleHash(getKeyDistributorBuckets(tableId, htd))));
}

Example 3 with RowKeyDistributorByHashPrefix

use of io.cdap.cdap.hbase.wd.RowKeyDistributorByHashPrefix in project cdap by caskdata.

the class HBaseTableUtilTest method testGetSplitKeys.

@Test
public void testGetSplitKeys() {
    int buckets = 16;
    AbstractRowKeyDistributor distributor = new RowKeyDistributorByHashPrefix(new RowKeyDistributorByHashPrefix.OneByteSimpleHash(buckets));
    // Number of splits will be no less than user asked. If splits > buckets, the number of splits will bumped to
    // next multiple of bucket that is no less than user splits requested.
    // it should return one key less than required splits count, because HBase will take care of the first automatically
    Assert.assertEquals(getSplitSize(buckets, 12) - 1, HBaseTableUtil.getSplitKeys(12, buckets, distributor).length);
    Assert.assertEquals(getSplitSize(buckets, 16) - 1, HBaseTableUtil.getSplitKeys(16, buckets, distributor).length);
    // at least #buckets - 1, but no less than user asked
    Assert.assertEquals(buckets - 1, HBaseTableUtil.getSplitKeys(6, buckets, distributor).length);
    Assert.assertEquals(buckets - 1, HBaseTableUtil.getSplitKeys(2, buckets, distributor).length);
    // "1" can be used for queue tables that we know are not "hot", so we do not pre-split in this case
    Assert.assertEquals(0, HBaseTableUtil.getSplitKeys(1, buckets, distributor).length);
    // allows up to 255 * 8 - 1 splits
    Assert.assertEquals(255 * buckets - 1, HBaseTableUtil.getSplitKeys(255 * buckets, buckets, distributor).length);
    try {
        HBaseTableUtil.getSplitKeys(256 * buckets, buckets, distributor);
        Assert.fail("getSplitKeys(256) should have thrown IllegalArgumentException");
    } catch (IllegalArgumentException e) {
    // expected
    }
    try {
        HBaseTableUtil.getSplitKeys(0, buckets, distributor);
        Assert.fail("getSplitKeys(0) should have thrown IllegalArgumentException");
    } catch (IllegalArgumentException e) {
    // expected
    }
}