Examples with HBaseDDLExecutor co.cask.cdap.spi.hbase.HBaseDDLExecutor used on opensource projects

Example 6 with HBaseDDLExecutor

use of co.cask.cdap.spi.hbase.HBaseDDLExecutor in project cdap by caskdata.

the class DatasetUpgrader method upgradeUserTables.

private Map<String, Future<?>> upgradeUserTables(final NamespaceMeta namespaceMeta, ExecutorService executor) throws Exception {
    Map<String, Future<?>> futures = new HashMap<>();
    String hBaseNamespace = hBaseTableUtil.getHBaseNamespace(namespaceMeta);
    try (HBaseDDLExecutor ddlExecutor = ddlExecutorFactory.get();
        HBaseAdmin hAdmin = new HBaseAdmin(hConf)) {
        for (final HTableDescriptor desc : hAdmin.listTableDescriptorsByNamespace(HTableNameConverter.encodeHBaseEntity(hBaseNamespace))) {
            Runnable runnable = new Runnable() {

                public void run() {
                    try {
                        impersonator.doAs(namespaceMeta.getNamespaceId(), new Callable<Void>() {

                            @Override
                            public Void call() throws Exception {
                                if (isCDAPUserTable(desc)) {
                                    upgradeUserTable(desc);
                                } else if (isStreamOrQueueTable(desc.getNameAsString())) {
                                    updateTableDesc(desc, ddlExecutor);
                                }
                                return null;
                            }
                        });
                    } catch (Exception e) {
                        throw new RuntimeException(e);
                    }
                }
            };
            Future<?> future = executor.submit(runnable);
            futures.put(desc.getNameAsString(), future);
        }
    }
    return futures;
}

Example 7 with HBaseDDLExecutor

use of co.cask.cdap.spi.hbase.HBaseDDLExecutor in project cdap by caskdata.

the class ConfigurationTable method write.

/**
   * Writes the {@link CConfiguration} instance as a new row to the HBase table.  The {@link Type} given is used as
   * the row key (allowing multiple configurations to be stored).  After the new configuration is written, this will
   * delete any configurations written with an earlier timestamp (to prevent removed values from being visible).
   * @param cConf The CConfiguration instance to store
   * @throws IOException If an error occurs while writing the configuration
   */
public void write(Type type, CConfiguration cConf) throws IOException {
    // must create the table if it doesn't exist
    HTable table = null;
    try (HBaseDDLExecutor ddlExecutor = new HBaseDDLExecutorFactory(cConf, hbaseConf).get()) {
        HBaseTableUtil tableUtil = new HBaseTableUtilFactory(cConf).get();
        TableId tableId = tableUtil.createHTableId(NamespaceId.SYSTEM, TABLE_NAME);
        ColumnFamilyDescriptorBuilder cfdBuilder = HBaseTableUtil.getColumnFamilyDescriptorBuilder(Bytes.toString(FAMILY), hbaseConf);
        TableDescriptorBuilder tdBuilder = HBaseTableUtil.getTableDescriptorBuilder(tableId, cConf).addColumnFamily(cfdBuilder.build());
        ddlExecutor.createTableIfNotExists(tdBuilder.build(), null);
        long now = System.currentTimeMillis();
        long previous = now - 1;
        byte[] typeBytes = Bytes.toBytes(type.name());
        LOG.info("Writing new config row with key " + type);
        // populate the configuration data
        table = tableUtil.createHTable(hbaseConf, tableId);
        table.setAutoFlush(false);
        Put p = new Put(typeBytes);
        for (Map.Entry<String, String> e : cConf) {
            p.add(FAMILY, Bytes.toBytes(e.getKey()), now, Bytes.toBytes(e.getValue()));
        }
        table.put(p);
        LOG.info("Deleting any configuration from " + previous + " or before");
        Delete d = new Delete(typeBytes);
        d.deleteFamily(FAMILY, previous);
        table.delete(d);
    } finally {
        if (table != null) {
            try {
                table.close();
            } catch (IOException ioe) {
                LOG.error("Error closing HBaseAdmin: " + ioe.getMessage(), ioe);
            }
        }
    }
}

Example 8 with HBaseDDLExecutor

use of co.cask.cdap.spi.hbase.HBaseDDLExecutor in project cdap by caskdata.

the class HBaseTableFactory method upgradeCoProcessor.

private void upgradeCoProcessor(TableId tableId, Class<? extends Coprocessor> coprocessor) throws IOException {
    try (HBaseDDLExecutor ddlExecutor = ddlExecutorFactory.get()) {
        HTableDescriptor tableDescriptor;
        try (HBaseAdmin admin = new HBaseAdmin(hConf)) {
            // If table doesn't exist, then skip upgrading coprocessor
            if (!tableUtil.tableExists(admin, tableId)) {
                LOG.debug("TMS Table {} was not found. Skip upgrading coprocessor.", tableId);
                return;
            }
            tableDescriptor = tableUtil.getHTableDescriptor(admin, tableId);
        }
        // Get cdap version from the table
        ProjectInfo.Version version = HBaseTableUtil.getVersion(tableDescriptor);
        if (version.compareTo(ProjectInfo.getVersion()) >= 0) {
            // If cdap has version has not changed or is greater, no need to update. Just enable it, in case
            // it has been disabled by the upgrade tool, and return
            LOG.info("Table '{}' has not changed and its version '{}' is same or greater than current CDAP version '{}'", tableId, version, ProjectInfo.getVersion());
            enableTable(ddlExecutor, tableId);
            return;
        }
        // create a new descriptor for the table update
        HTableDescriptorBuilder newDescriptor = tableUtil.buildHTableDescriptor(tableDescriptor);
        // Remove old coprocessor
        Map<String, HBaseTableUtil.CoprocessorInfo> coprocessorInfo = HBaseTableUtil.getCoprocessorInfo(tableDescriptor);
        for (Map.Entry<String, HBaseTableUtil.CoprocessorInfo> coprocessorEntry : coprocessorInfo.entrySet()) {
            newDescriptor.removeCoprocessor(coprocessorEntry.getValue().getClassName());
        }
        // Add new coprocessor
        CoprocessorDescriptor coprocessorDescriptor = coprocessorManager.getCoprocessorDescriptor(coprocessor, Coprocessor.PRIORITY_USER);
        Path path = coprocessorDescriptor.getPath() == null ? null : new Path(coprocessorDescriptor.getPath());
        newDescriptor.addCoprocessor(coprocessorDescriptor.getClassName(), path, coprocessorDescriptor.getPriority(), coprocessorDescriptor.getProperties());
        // Update CDAP version, table prefix
        HBaseTableUtil.setVersion(newDescriptor);
        HBaseTableUtil.setTablePrefix(newDescriptor, cConf);
        // Disable Table
        disableTable(ddlExecutor, tableId);
        tableUtil.modifyTable(ddlExecutor, newDescriptor.build());
        LOG.debug("Enabling table '{}'...", tableId);
        enableTable(ddlExecutor, tableId);
    }
    LOG.info("Table '{}' update completed.", tableId);
}

Example 9 with HBaseDDLExecutor

use of co.cask.cdap.spi.hbase.HBaseDDLExecutor in project cdap by caskdata.

the class HBaseTableFactory method createTable.

/**
   * Creates a new instance of {@link HTable} for the given {@link TableId}. If the hbase table doesn't
   * exist, a new one will be created with the given number of splits.
   */
private HTableWithRowKeyDistributor createTable(TableId tableId, int splits, Class<? extends Coprocessor> coprocessor) throws IOException {
    // Lookup the table descriptor from the cache first. If it is there, we assume the HBase table exists
    // Otherwise, attempt to create it.
    HTable hTable = null;
    HTableDescriptor htd = tableDescriptors.get(tableId);
    if (htd == null) {
        synchronized (this) {
            htd = tableDescriptors.get(tableId);
            if (htd == null) {
                boolean tableExists;
                try (HBaseAdmin admin = new HBaseAdmin(hConf)) {
                    tableExists = tableUtil.tableExists(admin, tableId);
                }
                // Create the table if the table doesn't exist
                try (HBaseDDLExecutor ddlExecutor = ddlExecutorFactory.get()) {
                    // If table exists, then skip creating coprocessor etc
                    if (!tableExists) {
                        TableId metadataTableId = tableUtil.createHTableId(NamespaceId.SYSTEM, cConf.get(Constants.MessagingSystem.METADATA_TABLE_NAME));
                        ColumnFamilyDescriptorBuilder cfdBuilder = HBaseTableUtil.getColumnFamilyDescriptorBuilder(Bytes.toString(COLUMN_FAMILY), hConf);
                        TableDescriptorBuilder tdBuilder = HBaseTableUtil.getTableDescriptorBuilder(tableId, cConf);
                        tdBuilder.addColumnFamily(cfdBuilder.build()).addProperty(Constants.MessagingSystem.HBASE_MESSAGING_TABLE_PREFIX_NUM_BYTES, Integer.toString(1)).addProperty(Constants.MessagingSystem.KEY_DISTRIBUTOR_BUCKETS_ATTR, Integer.toString(splits)).addProperty(Constants.MessagingSystem.HBASE_METADATA_TABLE_NAMESPACE, metadataTableId.getNamespace()).addCoprocessor(coprocessorManager.getCoprocessorDescriptor(coprocessor, Coprocessor.PRIORITY_USER));
                        // Set the key distributor size the same as the initial number of splits,
                        // essentially one bucket per split.
                        byte[][] splitKeys = HBaseTableUtil.getSplitKeys(splits, splits, new RowKeyDistributorByHashPrefix(new OneByteSimpleHash(splits)));
                        ddlExecutor.createTableIfNotExists(tdBuilder.build(), splitKeys);
                        hTable = tableUtil.createHTable(hConf, tableId);
                        htd = hTable.getTableDescriptor();
                        tableDescriptors.put(tableId, htd);
                    } else {
                        hTable = tableUtil.createHTable(hConf, tableId);
                        htd = hTable.getTableDescriptor();
                        tableDescriptors.put(tableId, htd);
                    }
                }
            }
        }
    }
    if (hTable == null) {
        hTable = tableUtil.createHTable(hConf, tableId);
    }
    hTable.setAutoFlushTo(false);
    return new HTableWithRowKeyDistributor(hTable, new RowKeyDistributorByHashPrefix(new OneByteSimpleHash(getKeyDistributorBuckets(tableId, htd))));
}

Example 10 with HBaseDDLExecutor

use of co.cask.cdap.spi.hbase.HBaseDDLExecutor in project cdap by caskdata.

the class HBaseTableFactory method createMetadataTable.

@Override
public MetadataTable createMetadataTable(String tableName) throws IOException {
    TableId tableId = tableUtil.createHTableId(NamespaceId.SYSTEM, tableName);
    HTable hTable = null;
    // If the table descriptor is in the cache, we assume the table exists.
    if (!tableDescriptors.containsKey(tableId)) {
        synchronized (this) {
            if (!tableDescriptors.containsKey(tableId)) {
                try (HBaseDDLExecutor ddlExecutor = ddlExecutorFactory.get()) {
                    ColumnFamilyDescriptorBuilder cfdBuilder = HBaseTableUtil.getColumnFamilyDescriptorBuilder(Bytes.toString(COLUMN_FAMILY), hConf);
                    TableDescriptorBuilder tdBuilder = HBaseTableUtil.getTableDescriptorBuilder(tableId, cConf).addColumnFamily(cfdBuilder.build());
                    ddlExecutor.createTableIfNotExists(tdBuilder.build(), null);
                    hTable = tableUtil.createHTable(hConf, tableId);
                    tableDescriptors.put(tableId, hTable.getTableDescriptor());
                }
            }
        }
    }
    if (hTable == null) {
        hTable = tableUtil.createHTable(hConf, tableId);
    }
    return new HBaseMetadataTable(tableUtil, hTable, COLUMN_FAMILY, cConf.getInt(Constants.MessagingSystem.HBASE_SCAN_CACHE_ROWS), createExceptionHandler(tableId));
}