Examples with TableDescriptorBuilder io.cdap.cdap.data2.util.hbase.TableDescriptorBuilder used on opensource projects

Example 1 with TableDescriptorBuilder

use of io.cdap.cdap.data2.util.hbase.TableDescriptorBuilder in project cdap by caskdata.

the class HBaseStreamFileConsumerFactory method create.

@Override
protected StreamConsumer create(TableId tableId, StreamConfig streamConfig, ConsumerConfig consumerConfig, StreamConsumerStateStore stateStore, StreamConsumerState beginConsumerState, FileReader<StreamEventOffset, Iterable<StreamFileOffset>> reader, @Nullable ReadFilter extraFilter) throws IOException {
    int splits = cConf.getInt(Constants.Stream.CONSUMER_TABLE_PRESPLITS);
    AbstractRowKeyDistributor distributor = new RowKeyDistributorByHashPrefix(new RowKeyDistributorByHashPrefix.OneByteSimpleHash(splits));
    byte[][] splitKeys = HBaseTableUtil.getSplitKeys(splits, splits, distributor);
    TableId hBaseTableId = tableUtil.createHTableId(new NamespaceId(tableId.getNamespace()), tableId.getTableName());
    TableDescriptorBuilder tdBuilder = HBaseTableUtil.getTableDescriptorBuilder(hBaseTableId, cConf);
    ColumnFamilyDescriptorBuilder cfdBuilder = HBaseTableUtil.getColumnFamilyDescriptorBuilder(Bytes.toString(QueueEntryRow.COLUMN_FAMILY), hConf);
    tdBuilder.addColumnFamily(cfdBuilder.build());
    tdBuilder.addProperty(QueueConstants.DISTRIBUTOR_BUCKETS, Integer.toString(splits));
    try (HBaseDDLExecutor ddlExecutor = ddlExecutorFactory.get()) {
        ddlExecutor.createTableIfNotExists(tdBuilder.build(), splitKeys);
    }
    HTable hTable = tableUtil.createHTable(hConf, hBaseTableId);
    hTable.setWriteBufferSize(Constants.Stream.HBASE_WRITE_BUFFER_SIZE);
    hTable.setAutoFlushTo(false);
    return new HBaseStreamFileConsumer(cConf, streamConfig, consumerConfig, tableUtil, hTable, reader, stateStore, beginConsumerState, extraFilter, createKeyDistributor(hTable.getTableDescriptor()));
}

Example 2 with TableDescriptorBuilder

use of io.cdap.cdap.data2.util.hbase.TableDescriptorBuilder in project cdap by caskdata.

the class HBaseStreamConsumerStateStoreFactory method create.

@Override
public synchronized StreamConsumerStateStore create(StreamConfig streamConfig) throws IOException {
    NamespaceId namespace = streamConfig.getStreamId().getParent();
    TableId streamStateStoreTableId = StreamUtils.getStateStoreTableId(namespace);
    TableId hbaseTableId = tableUtil.createHTableId(new NamespaceId(streamStateStoreTableId.getNamespace()), streamStateStoreTableId.getTableName());
    boolean tableExist;
    try (HBaseAdmin admin = new HBaseAdmin(hConf)) {
        tableExist = tableUtil.tableExists(admin, hbaseTableId);
    }
    if (!tableExist) {
        try (HBaseDDLExecutor ddlExecutor = ddlExecutorFactory.get()) {
            TableDescriptorBuilder tdBuilder = HBaseTableUtil.getTableDescriptorBuilder(hbaseTableId, cConf);
            ColumnFamilyDescriptorBuilder cfdBuilder = HBaseTableUtil.getColumnFamilyDescriptorBuilder(Bytes.toString(QueueEntryRow.COLUMN_FAMILY), hConf);
            tdBuilder.addColumnFamily(cfdBuilder.build());
            ddlExecutor.createTableIfNotExists(tdBuilder.build(), null);
        }
    }
    HTable hTable = tableUtil.createHTable(hConf, hbaseTableId);
    hTable.setWriteBufferSize(Constants.Stream.HBASE_WRITE_BUFFER_SIZE);
    hTable.setAutoFlushTo(false);
    return new HBaseStreamConsumerStateStore(streamConfig, hTable);
}

Example 3 with TableDescriptorBuilder

use of io.cdap.cdap.data2.util.hbase.TableDescriptorBuilder in project cdap by caskdata.

the class HBaseTableFactory method createMetadataTable.

@Override
public MetadataTable createMetadataTable(String tableName) throws IOException {
    TableId tableId = tableUtil.createHTableId(NamespaceId.SYSTEM, tableName);
    HTable hTable = null;
    // If the table descriptor is in the cache, we assume the table exists.
    if (!tableDescriptors.containsKey(tableId)) {
        synchronized (this) {
            if (!tableDescriptors.containsKey(tableId)) {
                try (HBaseDDLExecutor ddlExecutor = ddlExecutorFactory.get()) {
                    ColumnFamilyDescriptorBuilder cfdBuilder = HBaseTableUtil.getColumnFamilyDescriptorBuilder(Bytes.toString(COLUMN_FAMILY), hConf);
                    TableDescriptorBuilder tdBuilder = HBaseTableUtil.getTableDescriptorBuilder(tableId, cConf).addColumnFamily(cfdBuilder.build());
                    ddlExecutor.createTableIfNotExists(tdBuilder.build(), null);
                    hTable = tableUtil.createHTable(hConf, tableId);
                    tableDescriptors.put(tableId, hTable.getTableDescriptor());
                }
            }
        }
    }
    if (hTable == null) {
        hTable = tableUtil.createHTable(hConf, tableId);
    }
    return new HBaseMetadataTable(tableUtil, hTable, COLUMN_FAMILY, cConf.getInt(Constants.MessagingSystem.HBASE_SCAN_CACHE_ROWS), createExceptionHandler(tableId));
}

Example 4 with TableDescriptorBuilder

use of io.cdap.cdap.data2.util.hbase.TableDescriptorBuilder in project cdap by caskdata.

the class HBaseTableFactory method createTable.

/**
 * Creates a new instance of {@link HTable} for the given {@link TableId}. If the hbase table doesn't
 * exist, a new one will be created with the given number of splits.
 */
private HTableWithRowKeyDistributor createTable(TableId tableId, int splits, Class<? extends Coprocessor> coprocessor) throws IOException {
    // Lookup the table descriptor from the cache first. If it is there, we assume the HBase table exists
    // Otherwise, attempt to create it.
    HTable hTable = null;
    HTableDescriptor htd = tableDescriptors.get(tableId);
    if (htd == null) {
        synchronized (this) {
            htd = tableDescriptors.get(tableId);
            if (htd == null) {
                boolean tableExists;
                try (HBaseAdmin admin = new HBaseAdmin(hConf)) {
                    tableExists = tableUtil.tableExists(admin, tableId);
                }
                // Create the table if the table doesn't exist
                try (HBaseDDLExecutor ddlExecutor = ddlExecutorFactory.get()) {
                    // If table exists, then skip creating coprocessor etc
                    if (!tableExists) {
                        TableId metadataTableId = tableUtil.createHTableId(NamespaceId.SYSTEM, cConf.get(Constants.MessagingSystem.METADATA_TABLE_NAME));
                        ColumnFamilyDescriptorBuilder cfdBuilder = HBaseTableUtil.getColumnFamilyDescriptorBuilder(Bytes.toString(COLUMN_FAMILY), hConf);
                        TableDescriptorBuilder tdBuilder = HBaseTableUtil.getTableDescriptorBuilder(tableId, cConf).addColumnFamily(cfdBuilder.build()).addProperty(Constants.MessagingSystem.HBASE_MESSAGING_TABLE_PREFIX_NUM_BYTES, Integer.toString(1)).addProperty(Constants.MessagingSystem.KEY_DISTRIBUTOR_BUCKETS_ATTR, Integer.toString(splits)).addProperty(Constants.MessagingSystem.HBASE_METADATA_TABLE_NAMESPACE, metadataTableId.getNamespace()).addProperty(HTableDescriptor.SPLIT_POLICY, cConf.get(Constants.MessagingSystem.TABLE_HBASE_SPLIT_POLICY)).addCoprocessor(coprocessorManager.getCoprocessorDescriptor(coprocessor, Coprocessor.PRIORITY_USER));
                        // Set the key distributor size the same as the initial number of splits,
                        // essentially one bucket per split.
                        byte[][] splitKeys = HBaseTableUtil.getSplitKeys(splits, splits, new RowKeyDistributorByHashPrefix(new OneByteSimpleHash(splits)));
                        ddlExecutor.createTableIfNotExists(tdBuilder.build(), splitKeys);
                        hTable = tableUtil.createHTable(hConf, tableId);
                        htd = hTable.getTableDescriptor();
                        tableDescriptors.put(tableId, htd);
                    } else {
                        hTable = tableUtil.createHTable(hConf, tableId);
                        htd = hTable.getTableDescriptor();
                        tableDescriptors.put(tableId, htd);
                    }
                }
            }
        }
    }
    if (hTable == null) {
        hTable = tableUtil.createHTable(hConf, tableId);
    }
    hTable.setAutoFlushTo(false);
    return new HTableWithRowKeyDistributor(hTable, new RowKeyDistributorByHashPrefix(new OneByteSimpleHash(getKeyDistributorBuckets(tableId, htd))));
}

Example 5 with TableDescriptorBuilder

use of io.cdap.cdap.data2.util.hbase.TableDescriptorBuilder in project cdap by caskdata.

the class HBaseTableAdmin method create.

@Override
public void create() throws IOException {
    String columnFamily = Bytes.toString(TableProperties.getColumnFamilyBytes(spec.getProperties()));
    ColumnFamilyDescriptorBuilder cfdBuilder = HBaseTableUtil.getColumnFamilyDescriptorBuilder(columnFamily, hConf);
    if (TableProperties.getReadlessIncrementSupport(spec.getProperties())) {
        cfdBuilder.setMaxVersions(Integer.MAX_VALUE);
    } else if (DatasetsUtil.isTransactional(spec.getProperties())) {
        // NOTE: we cannot limit number of versions as there's no hard limit on # of excluded from read txs
        cfdBuilder.setMaxVersions(Integer.MAX_VALUE);
    } else {
        cfdBuilder.setMaxVersions(1);
    }
    cfdBuilder.setBloomType(ColumnFamilyDescriptor.BloomType.ROW);
    Long ttl = TableProperties.getTTL(spec.getProperties());
    if (ttl != null) {
        // convert ttl from seconds to milli-seconds
        ttl = TimeUnit.SECONDS.toMillis(ttl);
        cfdBuilder.addProperty(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
    }
    final TableDescriptorBuilder tdBuilder = HBaseTableUtil.getTableDescriptorBuilder(tableId, cConf);
    // if the dataset is configured for read-less increments, then set the table property to support upgrades
    boolean supportsReadlessIncrements = TableProperties.getReadlessIncrementSupport(spec.getProperties());
    if (supportsReadlessIncrements) {
        tdBuilder.addProperty(Table.PROPERTY_READLESS_INCREMENT, "true");
    }
    // if the dataset is configured to be non-transactional, then set the table property to support upgrades
    if (!DatasetsUtil.isTransactional(spec.getProperties())) {
        tdBuilder.addProperty(Constants.Dataset.TABLE_TX_DISABLED, "true");
        if (supportsReadlessIncrements) {
            // read-less increments CPs by default assume that table is transactional
            cfdBuilder.addProperty("dataset.table.readless.increment.transactional", "false");
        }
    }
    tdBuilder.addColumnFamily(cfdBuilder.build());
    CoprocessorJar coprocessorJar = createCoprocessorJar();
    for (Class<? extends Coprocessor> coprocessor : coprocessorJar.getCoprocessors()) {
        tdBuilder.addCoprocessor(coprocessorManager.getCoprocessorDescriptor(coprocessor, coprocessorJar.getPriority(coprocessor)));
    }
    byte[][] splits = null;
    String splitsProperty = spec.getProperty(PROPERTY_SPLITS);
    if (splitsProperty != null) {
        splits = GSON.fromJson(splitsProperty, byte[][].class);
    }
    // Disable split policy
    String splitsPolicy = spec.getProperty(SPLIT_POLICY);
    if (!Strings.isNullOrEmpty(splitsPolicy)) {
        tdBuilder.addProperty(HTableDescriptor.SPLIT_POLICY, splitsPolicy);
    }
    try (HBaseDDLExecutor ddlExecutor = ddlExecutorFactory.get()) {
        ddlExecutor.createTableIfNotExists(tdBuilder.build(), splits);
        try {
            Map<String, String> permissions = TableProperties.getTablePermissions(spec.getProperties());
            if (permissions != null && !permissions.isEmpty()) {
                tableUtil.grantPermissions(ddlExecutor, tableId, permissions);
            }
        } catch (IOException | RuntimeException e) {
            try {
                drop();
            } catch (Throwable t) {
                e.addSuppressed(t);
            }
            throw e;
        }
    }
}