Examples with HBaseDDLExecutor io.cdap.cdap.spi.hbase.HBaseDDLExecutor used on opensource projects

Example 1 with HBaseDDLExecutor

use of io.cdap.cdap.spi.hbase.HBaseDDLExecutor in project cdap by caskdata.

the class HBaseTableAdmin method create.

@Override
public void create() throws IOException {
    String columnFamily = Bytes.toString(TableProperties.getColumnFamilyBytes(spec.getProperties()));
    ColumnFamilyDescriptorBuilder cfdBuilder = HBaseTableUtil.getColumnFamilyDescriptorBuilder(columnFamily, hConf);
    if (TableProperties.getReadlessIncrementSupport(spec.getProperties())) {
        cfdBuilder.setMaxVersions(Integer.MAX_VALUE);
    } else if (DatasetsUtil.isTransactional(spec.getProperties())) {
        // NOTE: we cannot limit number of versions as there's no hard limit on # of excluded from read txs
        cfdBuilder.setMaxVersions(Integer.MAX_VALUE);
    } else {
        cfdBuilder.setMaxVersions(1);
    }
    cfdBuilder.setBloomType(ColumnFamilyDescriptor.BloomType.ROW);
    Long ttl = TableProperties.getTTL(spec.getProperties());
    if (ttl != null) {
        // convert ttl from seconds to milli-seconds
        ttl = TimeUnit.SECONDS.toMillis(ttl);
        cfdBuilder.addProperty(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
    }
    final TableDescriptorBuilder tdBuilder = HBaseTableUtil.getTableDescriptorBuilder(tableId, cConf);
    // if the dataset is configured for read-less increments, then set the table property to support upgrades
    boolean supportsReadlessIncrements = TableProperties.getReadlessIncrementSupport(spec.getProperties());
    if (supportsReadlessIncrements) {
        tdBuilder.addProperty(Table.PROPERTY_READLESS_INCREMENT, "true");
    }
    // if the dataset is configured to be non-transactional, then set the table property to support upgrades
    if (!DatasetsUtil.isTransactional(spec.getProperties())) {
        tdBuilder.addProperty(Constants.Dataset.TABLE_TX_DISABLED, "true");
        if (supportsReadlessIncrements) {
            // read-less increments CPs by default assume that table is transactional
            cfdBuilder.addProperty("dataset.table.readless.increment.transactional", "false");
        }
    }
    tdBuilder.addColumnFamily(cfdBuilder.build());
    CoprocessorJar coprocessorJar = createCoprocessorJar();
    for (Class<? extends Coprocessor> coprocessor : coprocessorJar.getCoprocessors()) {
        tdBuilder.addCoprocessor(coprocessorManager.getCoprocessorDescriptor(coprocessor, coprocessorJar.getPriority(coprocessor)));
    }
    byte[][] splits = null;
    String splitsProperty = spec.getProperty(PROPERTY_SPLITS);
    if (splitsProperty != null) {
        splits = GSON.fromJson(splitsProperty, byte[][].class);
    }
    // Disable split policy
    String splitsPolicy = spec.getProperty(SPLIT_POLICY);
    if (!Strings.isNullOrEmpty(splitsPolicy)) {
        tdBuilder.addProperty(HTableDescriptor.SPLIT_POLICY, splitsPolicy);
    }
    try (HBaseDDLExecutor ddlExecutor = ddlExecutorFactory.get()) {
        ddlExecutor.createTableIfNotExists(tdBuilder.build(), splits);
        try {
            Map<String, String> permissions = TableProperties.getTablePermissions(spec.getProperties());
            if (permissions != null && !permissions.isEmpty()) {
                tableUtil.grantPermissions(ddlExecutor, tableId, permissions);
            }
        } catch (IOException | RuntimeException e) {
            try {
                drop();
            } catch (Throwable t) {
                e.addSuppressed(t);
            }
            throw e;
        }
    }
}

Example 2 with HBaseDDLExecutor

use of io.cdap.cdap.spi.hbase.HBaseDDLExecutor in project cdap by caskdata.

the class DistributedStorageProviderNamespaceAdmin method delete.

@SuppressWarnings("ConstantConditions")
@Override
public void delete(NamespaceId namespaceId) throws IOException, ExploreException, SQLException {
    // delete namespace directory from filesystem
    super.delete(namespaceId);
    if (NamespaceId.DEFAULT.equals(namespaceId)) {
        return;
    }
    // delete HBase namespace
    NamespaceConfig namespaceConfig;
    try {
        namespaceConfig = namespaceQueryAdmin.get(namespaceId).getConfig();
    } catch (Exception ex) {
        throw new IOException("Could not fetch custom HBase mapping.", ex);
    }
    if (!Strings.isNullOrEmpty(namespaceConfig.getHbaseNamespace())) {
        // custom namespace mapping is set for HBase, hence don't do anything during delete since the lifecycle of the
        // namespace will be managed by the user
        LOG.debug("Custom HBase mapping {} was found while deleting {}. Hence skipping deletion of HBase namespace", namespaceConfig.getHbaseNamespace(), namespaceId);
        return;
    }
    // delete HBase namespace
    String namespace = tableUtil.getHBaseNamespace(namespaceId);
    try (HBaseDDLExecutor executor = hBaseDDLExecutorFactory.get()) {
        executor.deleteNamespaceIfExists(namespace);
    }
}

Example 3 with HBaseDDLExecutor

use of io.cdap.cdap.spi.hbase.HBaseDDLExecutor in project cdap by caskdata.

the class DistributedStorageProviderNamespaceAdmin method create.

@Override
public void create(NamespaceMeta namespaceMeta) throws IOException, ExploreException, SQLException {
    // create filesystem directory
    super.create(namespaceMeta);
    // skip namespace creation in HBase for default namespace
    if (NamespaceId.DEFAULT.equals(namespaceMeta.getNamespaceId())) {
        return;
    }
    // create HBase namespace and set group C(reate) permission if a group is configured
    String hbaseNamespace = tableUtil.getHBaseNamespace(namespaceMeta);
    if (Strings.isNullOrEmpty(namespaceMeta.getConfig().getHbaseNamespace())) {
        try (HBaseDDLExecutor executor = hBaseDDLExecutorFactory.get()) {
            boolean created = executor.createNamespaceIfNotExists(hbaseNamespace);
            if (namespaceMeta.getConfig().getGroupName() != null) {
                try {
                    executor.grantPermissions(hbaseNamespace, null, ImmutableMap.of("@" + namespaceMeta.getConfig().getGroupName(), "C"));
                } catch (IOException | RuntimeException e) {
                    // don't leave a partial state behind, as this fails the create(), the namespace should be removed
                    if (created) {
                        try {
                            executor.deleteNamespaceIfExists(hbaseNamespace);
                        } catch (Throwable t) {
                            e.addSuppressed(t);
                        }
                    }
                    throw e;
                }
            }
        } catch (Throwable t) {
            try {
                // if we failed to create a namespace in hbase then do clean up for above creations
                super.delete(namespaceMeta.getNamespaceId());
            } catch (Exception e) {
                t.addSuppressed(e);
            }
            throw t;
        }
    }
    try (HBaseAdmin admin = new HBaseAdmin(hConf)) {
        if (!tableUtil.hasNamespace(admin, hbaseNamespace)) {
            throw new IOException(String.format("HBase namespace '%s' specified for new namespace '%s' does not" + " exist. Please specify an existing HBase namespace.", hbaseNamespace, namespaceMeta.getName()));
        }
    }
}

Example 4 with HBaseDDLExecutor

use of io.cdap.cdap.spi.hbase.HBaseDDLExecutor in project cdap by caskdata.

the class ConfigurationWriter method createTableIfNecessary.

/**
 * Creates the configuration HBase table if it does not exist.
 */
@VisibleForTesting
void createTableIfNecessary() throws IOException {
    try (HBaseDDLExecutor ddlExecutor = new HBaseDDLExecutorFactory(cConf, hConf).get()) {
        HBaseTableUtil tableUtil = new HBaseTableUtilFactory(cConf).get();
        TableId tableId = tableUtil.createHTableId(NamespaceId.SYSTEM, TABLE_NAME);
        ColumnFamilyDescriptorBuilder cfdBuilder = HBaseTableUtil.getColumnFamilyDescriptorBuilder(Bytes.toString(FAMILY), hConf);
        TableDescriptorBuilder tdBuilder = HBaseTableUtil.getTableDescriptorBuilder(tableId, cConf).addColumnFamily(cfdBuilder.build());
        ddlExecutor.createTableIfNotExists(tdBuilder.build(), null);
    }
}

Example 5 with HBaseDDLExecutor

use of io.cdap.cdap.spi.hbase.HBaseDDLExecutor in project cdap by caskdata.

the class AbstractHBaseDataSetAdmin method updateTable.

/**
 * Performs update on a given HBase table. It will be updated if either its spec has
 * changed since the HBase table was created or updated, or if the CDAP version recorded
 * in the HTable descriptor is less than the current CDAP version.
 *
 * @param force forces update regardless of whether the table needs it.
 * @throws IOException If update failed.
 */
public void updateTable(boolean force) throws IOException {
    try (HBaseDDLExecutor ddlExecutor = ddlExecutorFactory.get()) {
        HTableDescriptor tableDescriptor;
        HTableDescriptorBuilder newDescriptor;
        try (HBaseAdmin admin = new HBaseAdmin(hConf)) {
            tableDescriptor = tableUtil.getHTableDescriptor(admin, tableId);
            // create a new descriptor for the table update
            newDescriptor = tableUtil.buildHTableDescriptor(tableDescriptor);
        }
        // update any table properties if necessary
        boolean needUpdate = needsUpdate(tableDescriptor, newDescriptor) || force;
        // Get the cdap version from the table
        ProjectInfo.Version version = HBaseTableUtil.getVersion(tableDescriptor);
        String hbaseVersion = HBaseTableUtil.getHBaseVersion(tableDescriptor);
        if (!needUpdate && hbaseVersion != null && hbaseVersion.equals(HBaseVersion.getVersionString()) && version.compareTo(ProjectInfo.getVersion()) >= 0) {
            // If neither the table spec nor the cdap version have changed, no need to update
            LOG.info("Table '{}' has not changed and its version '{}' is same or greater " + "than current CDAP version '{}'. The underlying HBase version {} has also not changed.", tableId, version, ProjectInfo.getVersion(), hbaseVersion);
            return;
        }
        // Generate the coprocessor jar
        CoprocessorJar coprocessorJar = createCoprocessorJar();
        Location jarLocation = coprocessorJar.getJarLocation();
        // Check if coprocessor upgrade is needed
        Map<String, HBaseTableUtil.CoprocessorInfo> coprocessorInfo = HBaseTableUtil.getCoprocessorInfo(tableDescriptor);
        // For all required coprocessors, check if they've need to be upgraded.
        for (Class<? extends Coprocessor> coprocessor : coprocessorJar.getCoprocessors()) {
            HBaseTableUtil.CoprocessorInfo info = coprocessorInfo.get(coprocessor.getName());
            if (info != null) {
                // The same coprocessor has been configured, check by the file name to see if they are the same.
                if (!jarLocation.getName().equals(info.getPath().getName())) {
                    // Remove old one and add the new one.
                    newDescriptor.removeCoprocessor(info.getClassName());
                    addCoprocessor(newDescriptor, coprocessor, coprocessorJar.getPriority(coprocessor));
                }
            } else {
                // The coprocessor is missing from the table, add it.
                addCoprocessor(newDescriptor, coprocessor, coprocessorJar.getPriority(coprocessor));
            }
        }
        // Removes all old coprocessors
        Set<String> coprocessorNames = ImmutableSet.copyOf(Iterables.transform(coprocessorJar.coprocessors, CLASS_TO_NAME));
        for (String remove : Sets.difference(coprocessorInfo.keySet(), coprocessorNames)) {
            newDescriptor.removeCoprocessor(remove);
        }
        HBaseTableUtil.setVersion(newDescriptor);
        HBaseTableUtil.setHBaseVersion(newDescriptor);
        HBaseTableUtil.setTablePrefix(newDescriptor, cConf);
        LOG.info("Updating table '{}'...", tableId);
        TableName tableName = HTableNameConverter.toTableName(cConf.get(Constants.Dataset.TABLE_PREFIX), tableId);
        boolean enableTable = false;
        try {
            ddlExecutor.disableTableIfEnabled(tableName.getNamespaceAsString(), tableName.getQualifierAsString());
            enableTable = true;
        } catch (TableNotEnabledException e) {
            // If the table is in cdap_system namespace enable it regardless so that they can be used later. See CDAP-7324
            if (isSystemTable()) {
                enableTable = true;
            } else {
                LOG.debug("Table '{}' was not enabled before update and will not be enabled after update.", tableId);
            }
        }
        tableUtil.modifyTable(ddlExecutor, newDescriptor.build());
        if (enableTable) {
            LOG.debug("Enabling table '{}'...", tableId);
            ddlExecutor.enableTableIfDisabled(tableName.getNamespaceAsString(), tableName.getQualifierAsString());
        }
    }
    LOG.info("Table '{}' update completed.", tableId);
}