Search in sources :

Example 81 with HBaseAdmin

use of org.apache.hadoop.hbase.client.HBaseAdmin in project cdap by caskdata.

the class HBase10CDHTableUpdater method createTableIfNotExists.

@Override
protected void createTableIfNotExists(Configuration conf) throws IOException {
    try (HBaseAdmin admin = new HBaseAdmin(conf)) {
        String tableName = StatusUtils.getReplicationStateTableName(conf);
        if (admin.tableExists(tableName)) {
            return;
        }
        HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
        htd.addFamily(new HColumnDescriptor(ReplicationConstants.ReplicationStatusTool.TIME_FAMILY));
        admin.createTable(htd);
        LOG.info("Created Table {}.", tableName);
    }
}
Also used : HBaseAdmin(org.apache.hadoop.hbase.client.HBaseAdmin) HColumnDescriptor(org.apache.hadoop.hbase.HColumnDescriptor) HTableDescriptor(org.apache.hadoop.hbase.HTableDescriptor)

Example 82 with HBaseAdmin

use of org.apache.hadoop.hbase.client.HBaseAdmin in project cdap by caskdata.

the class AbstractHBaseDataSetAdmin method updateTable.

/**
   * Performs update on a given HBase table. It will be updated if either its spec has
   * changed since the HBase table was created or updated, or if the CDAP version recorded
   * in the HTable descriptor is less than the current CDAP version.
   *
   * @param force forces update regardless of whether the table needs it.
   * @throws IOException If update failed.
   */
public void updateTable(boolean force) throws IOException {
    try (HBaseDDLExecutor ddlExecutor = ddlExecutorFactory.get()) {
        HTableDescriptor tableDescriptor;
        try (HBaseAdmin admin = new HBaseAdmin(hConf)) {
            tableDescriptor = tableUtil.getHTableDescriptor(admin, tableId);
        }
        // update any table properties if necessary
        boolean needUpdate = needsUpdate(tableDescriptor) || force;
        // Get the cdap version from the table
        ProjectInfo.Version version = HBaseTableUtil.getVersion(tableDescriptor);
        if (!needUpdate && version.compareTo(ProjectInfo.getVersion()) >= 0) {
            // If neither the table spec nor the cdap version have changed, no need to update
            LOG.info("Table '{}' has not changed and its version '{}' is same or greater " + "than current CDAP version '{}'", tableId, version, ProjectInfo.getVersion());
            return;
        }
        // create a new descriptor for the table update
        HTableDescriptorBuilder newDescriptor = tableUtil.buildHTableDescriptor(tableDescriptor);
        // Generate the coprocessor jar
        CoprocessorJar coprocessorJar = createCoprocessorJar();
        Location jarLocation = coprocessorJar.getJarLocation();
        // Check if coprocessor upgrade is needed
        Map<String, HBaseTableUtil.CoprocessorInfo> coprocessorInfo = HBaseTableUtil.getCoprocessorInfo(tableDescriptor);
        // For all required coprocessors, check if they've need to be upgraded.
        for (Class<? extends Coprocessor> coprocessor : coprocessorJar.getCoprocessors()) {
            HBaseTableUtil.CoprocessorInfo info = coprocessorInfo.get(coprocessor.getName());
            if (info != null) {
                // The same coprocessor has been configured, check by the file name to see if they are the same.
                if (!jarLocation.getName().equals(info.getPath().getName())) {
                    // Remove old one and add the new one.
                    newDescriptor.removeCoprocessor(info.getClassName());
                    addCoprocessor(newDescriptor, coprocessor, coprocessorJar.getPriority(coprocessor));
                }
            } else {
                // The coprocessor is missing from the table, add it.
                addCoprocessor(newDescriptor, coprocessor, coprocessorJar.getPriority(coprocessor));
            }
        }
        // Removes all old coprocessors
        Set<String> coprocessorNames = ImmutableSet.copyOf(Iterables.transform(coprocessorJar.coprocessors, CLASS_TO_NAME));
        for (String remove : Sets.difference(coprocessorInfo.keySet(), coprocessorNames)) {
            newDescriptor.removeCoprocessor(remove);
        }
        HBaseTableUtil.setVersion(newDescriptor);
        HBaseTableUtil.setTablePrefix(newDescriptor, cConf);
        LOG.info("Updating table '{}'...", tableId);
        TableName tableName = HTableNameConverter.toTableName(cConf.get(Constants.Dataset.TABLE_PREFIX), tableId);
        boolean enableTable = false;
        try {
            ddlExecutor.disableTableIfEnabled(tableName.getNamespaceAsString(), tableName.getQualifierAsString());
            enableTable = true;
        } catch (TableNotEnabledException e) {
            // If the table is in cdap_system namespace enable it regardless so that they can be used later. See CDAP-7324
            if (isSystemTable()) {
                enableTable = true;
            } else {
                LOG.debug("Table '{}' was not enabled before update and will not be enabled after update.", tableId);
            }
        }
        tableUtil.modifyTable(ddlExecutor, newDescriptor.build());
        if (enableTable) {
            LOG.debug("Enabling table '{}'...", tableId);
            ddlExecutor.enableTableIfDisabled(tableName.getNamespaceAsString(), tableName.getQualifierAsString());
        }
    }
    LOG.info("Table '{}' update completed.", tableId);
}
Also used : HBaseDDLExecutor(co.cask.cdap.spi.hbase.HBaseDDLExecutor) HTableDescriptorBuilder(co.cask.cdap.data2.util.hbase.HTableDescriptorBuilder) HBaseTableUtil(co.cask.cdap.data2.util.hbase.HBaseTableUtil) HTableDescriptor(org.apache.hadoop.hbase.HTableDescriptor) HBaseAdmin(org.apache.hadoop.hbase.client.HBaseAdmin) TableName(org.apache.hadoop.hbase.TableName) ProjectInfo(co.cask.cdap.common.utils.ProjectInfo) Location(org.apache.twill.filesystem.Location) TableNotEnabledException(org.apache.hadoop.hbase.TableNotEnabledException)

Example 83 with HBaseAdmin

use of org.apache.hadoop.hbase.client.HBaseAdmin in project cdap by caskdata.

the class DatasetUpgrader method upgradeUserTables.

private Map<String, Future<?>> upgradeUserTables(final NamespaceMeta namespaceMeta, ExecutorService executor) throws Exception {
    Map<String, Future<?>> futures = new HashMap<>();
    String hBaseNamespace = hBaseTableUtil.getHBaseNamespace(namespaceMeta);
    try (HBaseDDLExecutor ddlExecutor = ddlExecutorFactory.get();
        HBaseAdmin hAdmin = new HBaseAdmin(hConf)) {
        for (final HTableDescriptor desc : hAdmin.listTableDescriptorsByNamespace(HTableNameConverter.encodeHBaseEntity(hBaseNamespace))) {
            Runnable runnable = new Runnable() {

                public void run() {
                    try {
                        impersonator.doAs(namespaceMeta.getNamespaceId(), new Callable<Void>() {

                            @Override
                            public Void call() throws Exception {
                                if (isCDAPUserTable(desc)) {
                                    upgradeUserTable(desc);
                                } else if (isStreamOrQueueTable(desc.getNameAsString())) {
                                    updateTableDesc(desc, ddlExecutor);
                                }
                                return null;
                            }
                        });
                    } catch (Exception e) {
                        throw new RuntimeException(e);
                    }
                }
            };
            Future<?> future = executor.submit(runnable);
            futures.put(desc.getNameAsString(), future);
        }
    }
    return futures;
}
Also used : HBaseDDLExecutor(co.cask.cdap.spi.hbase.HBaseDDLExecutor) HashMap(java.util.HashMap) IOException(java.io.IOException) ExecutionException(java.util.concurrent.ExecutionException) HTableDescriptor(org.apache.hadoop.hbase.HTableDescriptor) HBaseAdmin(org.apache.hadoop.hbase.client.HBaseAdmin) Future(java.util.concurrent.Future)

Example 84 with HBaseAdmin

use of org.apache.hadoop.hbase.client.HBaseAdmin in project cdap by caskdata.

the class HBase11TableUpdater method createTableIfNotExists.

@Override
protected void createTableIfNotExists(Configuration conf) throws IOException {
    try (HBaseAdmin admin = new HBaseAdmin(conf)) {
        String tableName = StatusUtils.getReplicationStateTableName(conf);
        if (admin.tableExists(tableName)) {
            return;
        }
        HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
        htd.addFamily(new HColumnDescriptor(ReplicationConstants.ReplicationStatusTool.TIME_FAMILY));
        admin.createTable(htd);
        LOG.info("Created Table {}.", tableName);
    }
}
Also used : HBaseAdmin(org.apache.hadoop.hbase.client.HBaseAdmin) HColumnDescriptor(org.apache.hadoop.hbase.HColumnDescriptor) HTableDescriptor(org.apache.hadoop.hbase.HTableDescriptor)

Example 85 with HBaseAdmin

use of org.apache.hadoop.hbase.client.HBaseAdmin in project cdap by caskdata.

the class HBaseTableFactory method upgradeCoProcessor.

private void upgradeCoProcessor(TableId tableId, Class<? extends Coprocessor> coprocessor) throws IOException {
    try (HBaseDDLExecutor ddlExecutor = ddlExecutorFactory.get()) {
        HTableDescriptor tableDescriptor;
        try (HBaseAdmin admin = new HBaseAdmin(hConf)) {
            // If table doesn't exist, then skip upgrading coprocessor
            if (!tableUtil.tableExists(admin, tableId)) {
                LOG.debug("TMS Table {} was not found. Skip upgrading coprocessor.", tableId);
                return;
            }
            tableDescriptor = tableUtil.getHTableDescriptor(admin, tableId);
        }
        // Get cdap version from the table
        ProjectInfo.Version version = HBaseTableUtil.getVersion(tableDescriptor);
        if (version.compareTo(ProjectInfo.getVersion()) >= 0) {
            // If cdap has version has not changed or is greater, no need to update. Just enable it, in case
            // it has been disabled by the upgrade tool, and return
            LOG.info("Table '{}' has not changed and its version '{}' is same or greater than current CDAP version '{}'", tableId, version, ProjectInfo.getVersion());
            enableTable(ddlExecutor, tableId);
            return;
        }
        // create a new descriptor for the table update
        HTableDescriptorBuilder newDescriptor = tableUtil.buildHTableDescriptor(tableDescriptor);
        // Remove old coprocessor
        Map<String, HBaseTableUtil.CoprocessorInfo> coprocessorInfo = HBaseTableUtil.getCoprocessorInfo(tableDescriptor);
        for (Map.Entry<String, HBaseTableUtil.CoprocessorInfo> coprocessorEntry : coprocessorInfo.entrySet()) {
            newDescriptor.removeCoprocessor(coprocessorEntry.getValue().getClassName());
        }
        // Add new coprocessor
        CoprocessorDescriptor coprocessorDescriptor = coprocessorManager.getCoprocessorDescriptor(coprocessor, Coprocessor.PRIORITY_USER);
        Path path = coprocessorDescriptor.getPath() == null ? null : new Path(coprocessorDescriptor.getPath());
        newDescriptor.addCoprocessor(coprocessorDescriptor.getClassName(), path, coprocessorDescriptor.getPriority(), coprocessorDescriptor.getProperties());
        // Update CDAP version, table prefix
        HBaseTableUtil.setVersion(newDescriptor);
        HBaseTableUtil.setTablePrefix(newDescriptor, cConf);
        // Disable Table
        disableTable(ddlExecutor, tableId);
        tableUtil.modifyTable(ddlExecutor, newDescriptor.build());
        LOG.debug("Enabling table '{}'...", tableId);
        enableTable(ddlExecutor, tableId);
    }
    LOG.info("Table '{}' update completed.", tableId);
}
Also used : HBaseDDLExecutor(co.cask.cdap.spi.hbase.HBaseDDLExecutor) Path(org.apache.hadoop.fs.Path) HTableDescriptorBuilder(co.cask.cdap.data2.util.hbase.HTableDescriptorBuilder) HTableDescriptor(org.apache.hadoop.hbase.HTableDescriptor) HBaseAdmin(org.apache.hadoop.hbase.client.HBaseAdmin) ProjectInfo(co.cask.cdap.common.utils.ProjectInfo) CoprocessorDescriptor(co.cask.cdap.spi.hbase.CoprocessorDescriptor) Map(java.util.Map) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap)

Aggregations

HBaseAdmin (org.apache.hadoop.hbase.client.HBaseAdmin)180 Test (org.junit.Test)93 HTableDescriptor (org.apache.hadoop.hbase.HTableDescriptor)76 PhoenixConnection (org.apache.phoenix.jdbc.PhoenixConnection)72 HColumnDescriptor (org.apache.hadoop.hbase.HColumnDescriptor)70 Connection (java.sql.Connection)66 Properties (java.util.Properties)54 TableName (org.apache.hadoop.hbase.TableName)36 IOException (java.io.IOException)33 ResultSet (java.sql.ResultSet)27 BaseTest (org.apache.phoenix.query.BaseTest)27 HTable (org.apache.hadoop.hbase.client.HTable)26 SQLException (java.sql.SQLException)22 TestUtil.closeConnection (org.apache.phoenix.util.TestUtil.closeConnection)22 Put (org.apache.hadoop.hbase.client.Put)16 Configuration (org.apache.hadoop.conf.Configuration)13 HTableInterface (org.apache.hadoop.hbase.client.HTableInterface)13 PreparedStatement (java.sql.PreparedStatement)12 PhoenixIOException (org.apache.phoenix.exception.PhoenixIOException)12 HBaseConfiguration (org.apache.hadoop.hbase.HBaseConfiguration)9