Examples with Version co.cask.cdap.data2.dataset2.lib.table.hbase.MetricHBaseTableUtil.Version used on opensource projects

Example 1 with Version

use of co.cask.cdap.data2.dataset2.lib.table.hbase.MetricHBaseTableUtil.Version in project cdap by caskdata.

the class HBaseCheck method run.

@Override
public void run() {
    LOG.info("Checking HBase version.");
    HBaseTableUtil hBaseTableUtil;
    try {
        hBaseTableUtil = new HBaseTableUtilFactory(cConf).get();
    } catch (ProvisionException e) {
        throw new RuntimeException("Unsupported Hbase version " + HBaseVersion.getVersionString());
    }
    LOG.info("  HBase version successfully verified.");
    LOG.info("Checking HBase availability.");
    try (HConnection hbaseConnection = HConnectionManager.createConnection(hConf)) {
        hbaseConnection.listTables();
        LOG.info("  HBase availability successfully verified.");
    } catch (IOException e) {
        throw new RuntimeException("Unable to connect to HBase. " + "Please check that HBase is running and that the correct HBase configuration (hbase-site.xml) " + "and libraries are included in the CDAP master classpath.", e);
    }
    if (hConf.getBoolean("hbase.security.authorization", false)) {
        if (cConf.getBoolean(TxConstants.TransactionPruning.PRUNE_ENABLE)) {
            LOG.info("HBase authorization and transaction pruning are enabled. Checking global admin privileges for cdap.");
            try {
                boolean isGlobalAdmin = hBaseTableUtil.isGlobalAdmin(hConf);
                LOG.info("Global admin privileges check status: {}", isGlobalAdmin);
                if (isGlobalAdmin) {
                    return;
                }
                // if global admin was false then depend on the TX_PRUNE_ACL_CHECK value
                if (cConf.getBoolean(Constants.Startup.TX_PRUNE_ACL_CHECK, false)) {
                    LOG.info("Found {} to be set to true. Continuing with cdap master startup even though global admin check " + "returned false", Constants.Startup.TX_PRUNE_ACL_CHECK);
                    return;
                }
                StringBuilder builder = new StringBuilder("Transaction pruning is enabled and cdap does not have global " + "admin privileges in HBase. Global admin privileges for cdap " + "are required for transaction pruning. " + "Either disable transaction pruning or grant global admin " + "privilege to cdap in HBase or can override this " + "check by setting ");
                builder.append(Constants.Startup.TX_PRUNE_ACL_CHECK);
                builder.append(" in cdap-site.xml.");
                if (HBaseVersion.get().equals(HBaseVersion.Version.HBASE_96) || HBaseVersion.get().equals(HBaseVersion.Version.HBASE_98)) {
                    builder.append(" Detected HBase version ");
                    builder.append(HBaseVersion.get());
                    builder.append(" CDAP will not be able determine if it has global admin privilege in HBase.");
                    builder.append(" After granting global admin privilege please set ");
                    builder.append(Constants.Startup.TX_PRUNE_ACL_CHECK);
                }
                throw new RuntimeException(builder.toString());
            } catch (IOException e) {
                throw new RuntimeException("Unable to determines cdap privileges as global admin in HBase.");
            }
        }
    }
    LOG.info("Hbase authorization is disabled. Skipping global admin check for transaction pruning.");
}

Example 2 with Version

use of co.cask.cdap.data2.dataset2.lib.table.hbase.MetricHBaseTableUtil.Version in project cdap by caskdata.

the class HBaseTableFactory method upgradeCoProcessor.

private void upgradeCoProcessor(TableId tableId, Class<? extends Coprocessor> coprocessor) throws IOException {
    try (HBaseDDLExecutor ddlExecutor = ddlExecutorFactory.get()) {
        HTableDescriptor tableDescriptor;
        try (HBaseAdmin admin = new HBaseAdmin(hConf)) {
            // If table doesn't exist, then skip upgrading coprocessor
            if (!tableUtil.tableExists(admin, tableId)) {
                LOG.debug("TMS Table {} was not found. Skip upgrading coprocessor.", tableId);
                return;
            }
            tableDescriptor = tableUtil.getHTableDescriptor(admin, tableId);
        }
        // Get cdap version from the table
        ProjectInfo.Version version = HBaseTableUtil.getVersion(tableDescriptor);
        if (version.compareTo(ProjectInfo.getVersion()) >= 0) {
            // If cdap has version has not changed or is greater, no need to update. Just enable it, in case
            // it has been disabled by the upgrade tool, and return
            LOG.info("Table '{}' has not changed and its version '{}' is same or greater than current CDAP version '{}'", tableId, version, ProjectInfo.getVersion());
            enableTable(ddlExecutor, tableId);
            return;
        }
        // create a new descriptor for the table update
        HTableDescriptorBuilder newDescriptor = tableUtil.buildHTableDescriptor(tableDescriptor);
        // Remove old coprocessor
        Map<String, HBaseTableUtil.CoprocessorInfo> coprocessorInfo = HBaseTableUtil.getCoprocessorInfo(tableDescriptor);
        for (Map.Entry<String, HBaseTableUtil.CoprocessorInfo> coprocessorEntry : coprocessorInfo.entrySet()) {
            newDescriptor.removeCoprocessor(coprocessorEntry.getValue().getClassName());
        }
        // Add new coprocessor
        CoprocessorDescriptor coprocessorDescriptor = coprocessorManager.getCoprocessorDescriptor(coprocessor, Coprocessor.PRIORITY_USER);
        Path path = coprocessorDescriptor.getPath() == null ? null : new Path(coprocessorDescriptor.getPath());
        newDescriptor.addCoprocessor(coprocessorDescriptor.getClassName(), path, coprocessorDescriptor.getPriority(), coprocessorDescriptor.getProperties());
        // Update CDAP version, table prefix
        HBaseTableUtil.setVersion(newDescriptor);
        HBaseTableUtil.setTablePrefix(newDescriptor, cConf);
        // Disable Table
        disableTable(ddlExecutor, tableId);
        tableUtil.modifyTable(ddlExecutor, newDescriptor.build());
        LOG.debug("Enabling table '{}'...", tableId);
        enableTable(ddlExecutor, tableId);
    }
    LOG.info("Table '{}' update completed.", tableId);
}

Example 3 with Version

use of co.cask.cdap.data2.dataset2.lib.table.hbase.MetricHBaseTableUtil.Version in project cdap by caskdata.

the class MetricsDataMigrator method migrateMetricsTableFromVersion26.

private void migrateMetricsTableFromVersion26(Version version) throws DataMigrationException {
    for (String scope : scopes) {
        EntityTable entityTable = new EntityTable(getOrCreateMetricsTable(String.format("%s.%s", scope, entityTableName), DatasetProperties.EMPTY));
        String scopedMetricsTableName = String.format("%s.%s", scope, metricsTableName);
        MetricsTable metricsTable = getOrCreateMetricsTable(scopedMetricsTableName, DatasetProperties.EMPTY);
        System.out.println("Migrating Metrics Data from table : " + scopedMetricsTableName);
        migrateMetricsData(entityTable, metricsTable, scope, version);
    }
}

Example 4 with Version

use of co.cask.cdap.data2.dataset2.lib.table.hbase.MetricHBaseTableUtil.Version in project cdap by caskdata.

the class AbstractHBaseDataSetAdmin method updateTable.

/**
   * Performs update on a given HBase table. It will be updated if either its spec has
   * changed since the HBase table was created or updated, or if the CDAP version recorded
   * in the HTable descriptor is less than the current CDAP version.
   *
   * @param force forces update regardless of whether the table needs it.
   * @throws IOException If update failed.
   */
public void updateTable(boolean force) throws IOException {
    try (HBaseDDLExecutor ddlExecutor = ddlExecutorFactory.get()) {
        HTableDescriptor tableDescriptor;
        try (HBaseAdmin admin = new HBaseAdmin(hConf)) {
            tableDescriptor = tableUtil.getHTableDescriptor(admin, tableId);
        }
        // update any table properties if necessary
        boolean needUpdate = needsUpdate(tableDescriptor) || force;
        // Get the cdap version from the table
        ProjectInfo.Version version = HBaseTableUtil.getVersion(tableDescriptor);
        if (!needUpdate && version.compareTo(ProjectInfo.getVersion()) >= 0) {
            // If neither the table spec nor the cdap version have changed, no need to update
            LOG.info("Table '{}' has not changed and its version '{}' is same or greater " + "than current CDAP version '{}'", tableId, version, ProjectInfo.getVersion());
            return;
        }
        // create a new descriptor for the table update
        HTableDescriptorBuilder newDescriptor = tableUtil.buildHTableDescriptor(tableDescriptor);
        // Generate the coprocessor jar
        CoprocessorJar coprocessorJar = createCoprocessorJar();
        Location jarLocation = coprocessorJar.getJarLocation();
        // Check if coprocessor upgrade is needed
        Map<String, HBaseTableUtil.CoprocessorInfo> coprocessorInfo = HBaseTableUtil.getCoprocessorInfo(tableDescriptor);
        // For all required coprocessors, check if they've need to be upgraded.
        for (Class<? extends Coprocessor> coprocessor : coprocessorJar.getCoprocessors()) {
            HBaseTableUtil.CoprocessorInfo info = coprocessorInfo.get(coprocessor.getName());
            if (info != null) {
                // The same coprocessor has been configured, check by the file name to see if they are the same.
                if (!jarLocation.getName().equals(info.getPath().getName())) {
                    // Remove old one and add the new one.
                    newDescriptor.removeCoprocessor(info.getClassName());
                    addCoprocessor(newDescriptor, coprocessor, coprocessorJar.getPriority(coprocessor));
                }
            } else {
                // The coprocessor is missing from the table, add it.
                addCoprocessor(newDescriptor, coprocessor, coprocessorJar.getPriority(coprocessor));
            }
        }
        // Removes all old coprocessors
        Set<String> coprocessorNames = ImmutableSet.copyOf(Iterables.transform(coprocessorJar.coprocessors, CLASS_TO_NAME));
        for (String remove : Sets.difference(coprocessorInfo.keySet(), coprocessorNames)) {
            newDescriptor.removeCoprocessor(remove);
        }
        HBaseTableUtil.setVersion(newDescriptor);
        HBaseTableUtil.setTablePrefix(newDescriptor, cConf);
        LOG.info("Updating table '{}'...", tableId);
        TableName tableName = HTableNameConverter.toTableName(cConf.get(Constants.Dataset.TABLE_PREFIX), tableId);
        boolean enableTable = false;
        try {
            ddlExecutor.disableTableIfEnabled(tableName.getNamespaceAsString(), tableName.getQualifierAsString());
            enableTable = true;
        } catch (TableNotEnabledException e) {
            // If the table is in cdap_system namespace enable it regardless so that they can be used later. See CDAP-7324
            if (isSystemTable()) {
                enableTable = true;
            } else {
                LOG.debug("Table '{}' was not enabled before update and will not be enabled after update.", tableId);
            }
        }
        tableUtil.modifyTable(ddlExecutor, newDescriptor.build());
        if (enableTable) {
            LOG.debug("Enabling table '{}'...", tableId);
            ddlExecutor.enableTableIfDisabled(tableName.getNamespaceAsString(), tableName.getQualifierAsString());
        }
    }
    LOG.info("Table '{}' update completed.", tableId);
}

Example 5 with Version

use of co.cask.cdap.data2.dataset2.lib.table.hbase.MetricHBaseTableUtil.Version in project cdap by caskdata.

the class AppMetadataStore method getWorkflowToken.

public WorkflowToken getWorkflowToken(ProgramId workflowId, String workflowRunId) {
    Preconditions.checkArgument(ProgramType.WORKFLOW == workflowId.getType());
    // Workflow token is stored with following key:
    // [wft][namespace][app][version][WORKFLOW][workflowName][workflowRun]
    MDSKey key = getProgramKeyBuilder(TYPE_WORKFLOW_TOKEN, workflowId).add(workflowRunId).build();
    BasicWorkflowToken workflowToken = get(key, BasicWorkflowToken.class);
    // Check without the version string only for default version
    if (!upgradeComplete.get() && workflowToken == null && workflowId.getVersion().equals(ApplicationId.DEFAULT_VERSION)) {
        key = getVersionLessProgramKeyBuilder(TYPE_WORKFLOW_TOKEN, workflowId).add(workflowRunId).build();
        workflowToken = get(key, BasicWorkflowToken.class);
    }
    if (workflowToken == null) {
        LOG.debug("No workflow token available for workflow: {}, runId: {}", workflowId, workflowRunId);
        // Its ok to not allow any updates by returning a 0 size token.
        return new BasicWorkflowToken(0);
    }
    return workflowToken;
}