Examples with HBaseAdmin org.apache.hadoop.hbase.client.HBaseAdmin used on opensource projects

Example 56 with HBaseAdmin

use of org.apache.hadoop.hbase.client.HBaseAdmin in project phoenix by apache.

the class TestUtil method doMajorCompaction.

/**
     * Runs a major compaction, and then waits until the compaction is complete before returning.
     *
     * @param tableName name of the table to be compacted
     */
public static void doMajorCompaction(Connection conn, String tableName) throws Exception {
    tableName = SchemaUtil.normalizeIdentifier(tableName);
    // We simply write a marker row, request a major compaction, and then wait until the marker
    // row is gone
    PhoenixConnection pconn = conn.unwrap(PhoenixConnection.class);
    PTable table = pconn.getTable(new PTableKey(pconn.getTenantId(), tableName));
    ConnectionQueryServices services = conn.unwrap(PhoenixConnection.class).getQueryServices();
    MutationState mutationState = pconn.getMutationState();
    if (table.isTransactional()) {
        mutationState.startTransaction();
    }
    try (HTableInterface htable = mutationState.getHTable(table)) {
        byte[] markerRowKey = Bytes.toBytes("TO_DELETE");
        Put put = new Put(markerRowKey);
        put.add(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, QueryConstants.EMPTY_COLUMN_VALUE_BYTES, QueryConstants.EMPTY_COLUMN_VALUE_BYTES);
        htable.put(put);
        Delete delete = new Delete(markerRowKey);
        delete.deleteColumn(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, QueryConstants.EMPTY_COLUMN_VALUE_BYTES);
        htable.delete(delete);
        htable.close();
        if (table.isTransactional()) {
            mutationState.commit();
        }
        HBaseAdmin hbaseAdmin = services.getAdmin();
        hbaseAdmin.flush(tableName);
        hbaseAdmin.majorCompact(tableName);
        hbaseAdmin.close();
        boolean compactionDone = false;
        while (!compactionDone) {
            Thread.sleep(6000L);
            Scan scan = new Scan();
            scan.setStartRow(markerRowKey);
            scan.setStopRow(Bytes.add(markerRowKey, new byte[] { 0 }));
            scan.setRaw(true);
            try (HTableInterface htableForRawScan = services.getTable(Bytes.toBytes(tableName))) {
                ResultScanner scanner = htableForRawScan.getScanner(scan);
                List<Result> results = Lists.newArrayList(scanner);
                LOG.info("Results: " + results);
                compactionDone = results.isEmpty();
                scanner.close();
            }
            LOG.info("Compaction done: " + compactionDone);
            // need to run compaction after the next txn snapshot has been written so that compaction can remove deleted rows
            if (!compactionDone && table.isTransactional()) {
                hbaseAdmin = services.getAdmin();
                hbaseAdmin.flush(tableName);
                hbaseAdmin.majorCompact(tableName);
                hbaseAdmin.close();
            }
        }
    }
}

Example 57 with HBaseAdmin

use of org.apache.hadoop.hbase.client.HBaseAdmin in project phoenix by apache.

the class ConnectionQueryServicesImpl method upgradeSystemTables.

/**
     * There is no other locking needed here since only one connection (on the same or different JVM) will be able to
     * acquire the upgrade mutex via {@link #acquireUpgradeMutex(long, byte[])}.
     */
@Override
public void upgradeSystemTables(final String url, final Properties props) throws SQLException {
    PhoenixConnection metaConnection = null;
    boolean success = false;
    String snapshotName = null;
    String sysCatalogTableName = null;
    SQLException toThrow = null;
    boolean acquiredMutexLock = false;
    byte[] mutexRowKey = SchemaUtil.getTableKey(null, PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA, PhoenixDatabaseMetaData.SYSTEM_CATALOG_TABLE);
    boolean snapshotCreated = false;
    try {
        if (!ConnectionQueryServicesImpl.this.upgradeRequired.get()) {
            throw new UpgradeNotRequiredException();
        }
        Properties scnProps = PropertiesUtil.deepCopy(props);
        scnProps.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP));
        scnProps.remove(PhoenixRuntime.TENANT_ID_ATTRIB);
        String globalUrl = JDBCUtil.removeProperty(url, PhoenixRuntime.TENANT_ID_ATTRIB);
        metaConnection = new PhoenixConnection(ConnectionQueryServicesImpl.this, globalUrl, scnProps, newEmptyMetaData());
        metaConnection.setRunningUpgrade(true);
        try {
            metaConnection.createStatement().executeUpdate(QueryConstants.CREATE_TABLE_METADATA);
        } catch (NewerTableAlreadyExistsException ignore) {
        // Ignore, as this will happen if the SYSTEM.CATALOG already exists at this fixed
        // timestamp. A TableAlreadyExistsException is not thrown, since the table only exists
        // *after* this fixed timestamp.
        } catch (TableAlreadyExistsException e) {
            long currentServerSideTableTimeStamp = e.getTable().getTimeStamp();
            sysCatalogTableName = e.getTable().getPhysicalName().getString();
            if (currentServerSideTableTimeStamp < MIN_SYSTEM_TABLE_TIMESTAMP && (acquiredMutexLock = acquireUpgradeMutex(currentServerSideTableTimeStamp, mutexRowKey))) {
                snapshotName = getSysCatalogSnapshotName(currentServerSideTableTimeStamp);
                createSnapshot(snapshotName, sysCatalogTableName);
                snapshotCreated = true;
            }
            String columnsToAdd = "";
            // include any new columns we've added.
            if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_3_0) {
                // We know that we always need to add the STORE_NULLS column for 4.3 release
                columnsToAdd = addColumn(columnsToAdd, PhoenixDatabaseMetaData.STORE_NULLS + " " + PBoolean.INSTANCE.getSqlTypeName());
                try (HBaseAdmin admin = getAdmin()) {
                    HTableDescriptor[] localIndexTables = admin.listTables(MetaDataUtil.LOCAL_INDEX_TABLE_PREFIX + ".*");
                    for (HTableDescriptor table : localIndexTables) {
                        if (table.getValue(MetaDataUtil.PARENT_TABLE_KEY) == null && table.getValue(MetaDataUtil.IS_LOCAL_INDEX_TABLE_PROP_NAME) != null) {
                            table.setValue(MetaDataUtil.PARENT_TABLE_KEY, MetaDataUtil.getLocalIndexUserTableName(table.getNameAsString()));
                            // Explicitly disable, modify and enable the table to ensure
                            // co-location of data and index regions. If we just modify the
                            // table descriptor when online schema change enabled may reopen 
                            // the region in same region server instead of following data region.
                            admin.disableTable(table.getTableName());
                            admin.modifyTable(table.getTableName(), table);
                            admin.enableTable(table.getTableName());
                        }
                    }
                }
            }
            // the column names that have been added to SYSTEM.CATALOG since 4.0.
            if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_1_0) {
                columnsToAdd = addColumn(columnsToAdd, PhoenixDatabaseMetaData.INDEX_TYPE + " " + PUnsignedTinyint.INSTANCE.getSqlTypeName() + ", " + PhoenixDatabaseMetaData.INDEX_DISABLE_TIMESTAMP + " " + PLong.INSTANCE.getSqlTypeName());
            }
            // If we have some new columns from 4.1-4.3 to add, add them now.
            if (!columnsToAdd.isEmpty()) {
                // Ugh..need to assign to another local variable to keep eclipse happy.
                PhoenixConnection newMetaConnection = addColumnsIfNotExists(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG, MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_3_0, columnsToAdd);
                metaConnection = newMetaConnection;
            }
            if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_5_0) {
                columnsToAdd = PhoenixDatabaseMetaData.BASE_COLUMN_COUNT + " " + PInteger.INSTANCE.getSqlTypeName();
                try {
                    metaConnection = addColumn(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG, MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_5_0, columnsToAdd, false);
                    upgradeTo4_5_0(metaConnection);
                } catch (ColumnAlreadyExistsException ignored) {
                    /*
                         * Upgrade to 4.5 is a slightly special case. We use the fact that the
                         * column BASE_COLUMN_COUNT is already part of the meta-data schema as the
                         * signal that the server side upgrade has finished or is in progress.
                         */
                    logger.debug("No need to run 4.5 upgrade");
                }
                Properties p = PropertiesUtil.deepCopy(metaConnection.getClientInfo());
                p.remove(PhoenixRuntime.CURRENT_SCN_ATTRIB);
                p.remove(PhoenixRuntime.TENANT_ID_ATTRIB);
                PhoenixConnection conn = new PhoenixConnection(ConnectionQueryServicesImpl.this, metaConnection.getURL(), p, metaConnection.getMetaDataCache());
                try {
                    List<String> tablesNeedingUpgrade = UpgradeUtil.getPhysicalTablesWithDescRowKey(conn);
                    if (!tablesNeedingUpgrade.isEmpty()) {
                        logger.warn("The following tables require upgrade due to a bug causing the row key to be incorrect for descending columns and ascending BINARY columns (PHOENIX-2067 and PHOENIX-2120):\n" + Joiner.on(' ').join(tablesNeedingUpgrade) + "\nTo upgrade issue the \"bin/psql.py -u\" command.");
                    }
                    List<String> unsupportedTables = UpgradeUtil.getPhysicalTablesWithDescVarbinaryRowKey(conn);
                    if (!unsupportedTables.isEmpty()) {
                        logger.warn("The following tables use an unsupported VARBINARY DESC construct and need to be changed:\n" + Joiner.on(' ').join(unsupportedTables));
                    }
                } catch (Exception ex) {
                    logger.error("Unable to determine tables requiring upgrade due to PHOENIX-2067", ex);
                } finally {
                    conn.close();
                }
            }
            // parts we haven't yet done).
            if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_6_0) {
                columnsToAdd = PhoenixDatabaseMetaData.IS_ROW_TIMESTAMP + " " + PBoolean.INSTANCE.getSqlTypeName();
                metaConnection = addColumnsIfNotExists(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG, MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_6_0, columnsToAdd);
            }
            if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0) {
                // Drop old stats table so that new stats table is created
                metaConnection = dropStatsTable(metaConnection, MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0 - 4);
                metaConnection = addColumnsIfNotExists(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG, MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0 - 3, PhoenixDatabaseMetaData.TRANSACTIONAL + " " + PBoolean.INSTANCE.getSqlTypeName());
                metaConnection = addColumnsIfNotExists(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG, MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0 - 2, PhoenixDatabaseMetaData.UPDATE_CACHE_FREQUENCY + " " + PLong.INSTANCE.getSqlTypeName());
                metaConnection = setImmutableTableIndexesImmutable(metaConnection, MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0 - 1);
                metaConnection = updateSystemCatalogTimestamp(metaConnection, MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0);
                ConnectionQueryServicesImpl.this.removeTable(null, PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME, null, MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0);
                clearCache();
            }
            if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_8_0) {
                metaConnection = addColumnsIfNotExists(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG, MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_8_0 - 2, PhoenixDatabaseMetaData.IS_NAMESPACE_MAPPED + " " + PBoolean.INSTANCE.getSqlTypeName());
                metaConnection = addColumnsIfNotExists(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG, MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_8_0 - 1, PhoenixDatabaseMetaData.AUTO_PARTITION_SEQ + " " + PVarchar.INSTANCE.getSqlTypeName());
                metaConnection = addColumnsIfNotExists(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG, MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_8_0, PhoenixDatabaseMetaData.APPEND_ONLY_SCHEMA + " " + PBoolean.INSTANCE.getSqlTypeName());
                metaConnection = UpgradeUtil.disableViewIndexes(metaConnection);
                if (getProps().getBoolean(QueryServices.LOCAL_INDEX_CLIENT_UPGRADE_ATTRIB, QueryServicesOptions.DEFAULT_LOCAL_INDEX_CLIENT_UPGRADE)) {
                    metaConnection = UpgradeUtil.upgradeLocalIndexes(metaConnection);
                }
                ConnectionQueryServicesImpl.this.removeTable(null, PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME, null, MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_8_0);
                clearCache();
            }
            if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_9_0) {
                metaConnection = addColumnsIfNotExists(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG, MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_9_0, PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH + " " + PLong.INSTANCE.getSqlTypeName());
                ConnectionQueryServicesImpl.this.removeTable(null, PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME, null, MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_9_0);
                clearCache();
            }
            if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_10_0) {
                metaConnection = addColumnQualifierColumn(metaConnection, MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_10_0 - 3);
                metaConnection = addColumnsIfNotExists(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG, MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_10_0 - 2, PhoenixDatabaseMetaData.IMMUTABLE_STORAGE_SCHEME + " " + PTinyint.INSTANCE.getSqlTypeName());
                metaConnection = addColumnsIfNotExists(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG, MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_10_0 - 1, PhoenixDatabaseMetaData.ENCODING_SCHEME + " " + PTinyint.INSTANCE.getSqlTypeName());
                metaConnection = addColumnsIfNotExists(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG, MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_10_0, PhoenixDatabaseMetaData.COLUMN_QUALIFIER_COUNTER + " " + PInteger.INSTANCE.getSqlTypeName());
                ConnectionQueryServicesImpl.this.removeTable(null, PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME, null, MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_10_0);
                clearCache();
            }
            if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_11_0) {
                metaConnection = addColumnsIfNotExists(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG, MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_11_0, PhoenixDatabaseMetaData.USE_STATS_FOR_PARALLELIZATION + " " + PBoolean.INSTANCE.getSqlTypeName());
                addParentToChildLinks(metaConnection);
            }
        }
        int nSaltBuckets = ConnectionQueryServicesImpl.this.props.getInt(QueryServices.SEQUENCE_SALT_BUCKETS_ATTRIB, QueryServicesOptions.DEFAULT_SEQUENCE_TABLE_SALT_BUCKETS);
        try {
            String createSequenceTable = Sequence.getCreateTableStatement(nSaltBuckets);
            metaConnection.createStatement().executeUpdate(createSequenceTable);
            nSequenceSaltBuckets = nSaltBuckets;
        } catch (NewerTableAlreadyExistsException e) {
            // Ignore, as this will happen if the SYSTEM.SEQUENCE already exists at this fixed
            // timestamp.
            // A TableAlreadyExistsException is not thrown, since the table only exists *after* this
            // fixed timestamp.
            nSequenceSaltBuckets = getSaltBuckets(e);
        } catch (TableAlreadyExistsException e) {
            // This will occur if we have an older SYSTEM.SEQUENCE and we need to update it to
            // include
            // any new columns we've added.
            long currentServerSideTableTimeStamp = e.getTable().getTimeStamp();
            if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_1_0) {
                // If the table time stamp is before 4.1.0 then we need to add below columns
                // to the SYSTEM.SEQUENCE table.
                String columnsToAdd = PhoenixDatabaseMetaData.MIN_VALUE + " " + PLong.INSTANCE.getSqlTypeName() + ", " + PhoenixDatabaseMetaData.MAX_VALUE + " " + PLong.INSTANCE.getSqlTypeName() + ", " + PhoenixDatabaseMetaData.CYCLE_FLAG + " " + PBoolean.INSTANCE.getSqlTypeName() + ", " + PhoenixDatabaseMetaData.LIMIT_REACHED_FLAG + " " + PBoolean.INSTANCE.getSqlTypeName();
                addColumnsIfNotExists(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG, MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP, columnsToAdd);
            }
            // If the table timestamp is before 4.2.1 then run the upgrade script
            if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_2_1) {
                if (UpgradeUtil.upgradeSequenceTable(metaConnection, nSaltBuckets, e.getTable())) {
                    metaConnection.removeTable(null, PhoenixDatabaseMetaData.SYSTEM_SEQUENCE_SCHEMA, PhoenixDatabaseMetaData.SYSTEM_SEQUENCE_TABLE, MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP);
                    clearTableFromCache(ByteUtil.EMPTY_BYTE_ARRAY, PhoenixDatabaseMetaData.SYSTEM_SEQUENCE_SCHEMA_BYTES, PhoenixDatabaseMetaData.SYSTEM_SEQUENCE_TABLE_BYTES, MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP);
                    clearTableRegionCache(PhoenixDatabaseMetaData.SYSTEM_SEQUENCE_NAME_BYTES);
                }
                nSequenceSaltBuckets = nSaltBuckets;
            } else {
                nSequenceSaltBuckets = getSaltBuckets(e);
            }
        }
        try {
            metaConnection.createStatement().executeUpdate(QueryConstants.CREATE_STATS_TABLE_METADATA);
        } catch (NewerTableAlreadyExistsException ignore) {
        } catch (TableAlreadyExistsException e) {
            long currentServerSideTableTimeStamp = e.getTable().getTimeStamp();
            if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_3_0) {
                metaConnection = addColumnsIfNotExists(metaConnection, SYSTEM_STATS_NAME, MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP, PhoenixDatabaseMetaData.GUIDE_POSTS_ROW_COUNT + " " + PLong.INSTANCE.getSqlTypeName());
            }
            if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_9_0) {
                // The COLUMN_FAMILY column should be nullable as we create a row in it without
                // any column family to mark when guideposts were last collected.
                metaConnection = removeNotNullConstraint(metaConnection, PhoenixDatabaseMetaData.SYSTEM_SCHEMA_NAME, PhoenixDatabaseMetaData.SYSTEM_STATS_TABLE, MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_9_0, PhoenixDatabaseMetaData.COLUMN_FAMILY);
                ConnectionQueryServicesImpl.this.removeTable(null, PhoenixDatabaseMetaData.SYSTEM_STATS_NAME, null, MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_9_0);
                clearCache();
            }
        }
        try {
            metaConnection.createStatement().executeUpdate(QueryConstants.CREATE_FUNCTION_METADATA);
        } catch (NewerTableAlreadyExistsException e) {
        } catch (TableAlreadyExistsException e) {
        }
        if (SchemaUtil.isNamespaceMappingEnabled(PTableType.SYSTEM, ConnectionQueryServicesImpl.this.getProps())) {
            try {
                metaConnection.createStatement().executeUpdate("CREATE SCHEMA IF NOT EXISTS " + PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA);
            } catch (NewerSchemaAlreadyExistsException e) {
            }
        }
        ConnectionQueryServicesImpl.this.upgradeRequired.set(false);
        success = true;
    } catch (UpgradeInProgressException | UpgradeNotRequiredException e) {
        // don't set it as initializationException because otherwise client won't be able to retry
        throw e;
    } catch (Exception e) {
        if (e instanceof SQLException) {
            toThrow = (SQLException) e;
        } else {
            // wrap every other exception into a SQLException
            toThrow = new SQLException(e);
        }
    } finally {
        try {
            if (metaConnection != null) {
                metaConnection.close();
            }
        } catch (SQLException e) {
            if (toThrow != null) {
                toThrow.setNextException(e);
            } else {
                toThrow = e;
            }
        } finally {
            try {
                if (snapshotCreated) {
                    restoreFromSnapshot(sysCatalogTableName, snapshotName, success);
                }
            } catch (SQLException e) {
                if (toThrow != null) {
                    toThrow.setNextException(e);
                } else {
                    toThrow = e;
                }
            } finally {
                if (acquiredMutexLock) {
                    releaseUpgradeMutex(mutexRowKey);
                }
            }
            if (toThrow != null) {
                throw toThrow;
            }
        }
    }
}

Example 58 with HBaseAdmin

use of org.apache.hadoop.hbase.client.HBaseAdmin in project phoenix by apache.

the class ConnectionQueryServicesImpl method init.

@Override
public void init(final String url, final Properties props) throws SQLException {
    try {
        PhoenixContextExecutor.call(new Callable<Void>() {

            @Override
            public Void call() throws Exception {
                if (initialized) {
                    if (initializationException != null) {
                        // Throw previous initialization exception, as we won't resuse this instance
                        throw initializationException;
                    }
                    return null;
                }
                synchronized (ConnectionQueryServicesImpl.this) {
                    if (initialized) {
                        if (initializationException != null) {
                            // Throw previous initialization exception, as we won't resuse this instance
                            throw initializationException;
                        }
                        return null;
                    }
                    checkClosed();
                    boolean hConnectionEstablished = false;
                    boolean success = false;
                    try {
                        GLOBAL_QUERY_SERVICES_COUNTER.increment();
                        logger.info("An instance of ConnectionQueryServices was created.");
                        openConnection();
                        hConnectionEstablished = true;
                        boolean isDoNotUpgradePropSet = UpgradeUtil.isNoUpgradeSet(props);
                        try (HBaseAdmin admin = getAdmin()) {
                            boolean mappedSystemCatalogExists = admin.tableExists(SchemaUtil.getPhysicalTableName(SYSTEM_CATALOG_NAME_BYTES, true));
                            if (SchemaUtil.isNamespaceMappingEnabled(PTableType.SYSTEM, ConnectionQueryServicesImpl.this.getProps())) {
                                if (admin.tableExists(SYSTEM_CATALOG_NAME_BYTES)) {
                                    //check if the server is already updated and have namespace config properly set. 
                                    checkClientServerCompatibility(SYSTEM_CATALOG_NAME_BYTES);
                                }
                                ensureSystemTablesUpgraded(ConnectionQueryServicesImpl.this.getProps());
                            } else if (mappedSystemCatalogExists) {
                                throw new SQLExceptionInfo.Builder(SQLExceptionCode.INCONSISTENET_NAMESPACE_MAPPING_PROPERTIES).setMessage("Cannot initiate connection as " + SchemaUtil.getPhysicalTableName(SYSTEM_CATALOG_NAME_BYTES, true) + " is found but client does not have " + IS_NAMESPACE_MAPPING_ENABLED + " enabled").build().buildException();
                            }
                            createSysMutexTable(admin);
                        }
                        Properties scnProps = PropertiesUtil.deepCopy(props);
                        scnProps.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP));
                        scnProps.remove(PhoenixRuntime.TENANT_ID_ATTRIB);
                        String globalUrl = JDBCUtil.removeProperty(url, PhoenixRuntime.TENANT_ID_ATTRIB);
                        try (PhoenixConnection metaConnection = new PhoenixConnection(ConnectionQueryServicesImpl.this, globalUrl, scnProps, newEmptyMetaData())) {
                            try {
                                metaConnection.createStatement().executeUpdate(QueryConstants.CREATE_TABLE_METADATA);
                            } catch (NewerTableAlreadyExistsException ignore) {
                            // Ignore, as this will happen if the SYSTEM.CATALOG already exists at this fixed
                            // timestamp. A TableAlreadyExistsException is not thrown, since the table only exists
                            // *after* this fixed timestamp.
                            } catch (TableAlreadyExistsException e) {
                                long currentServerSideTableTimeStamp = e.getTable().getTimeStamp();
                                if (currentServerSideTableTimeStamp < MIN_SYSTEM_TABLE_TIMESTAMP) {
                                    ConnectionQueryServicesImpl.this.upgradeRequired.set(true);
                                }
                            } catch (PhoenixIOException e) {
                                if (!Iterables.isEmpty(Iterables.filter(Throwables.getCausalChain(e), AccessDeniedException.class))) {
                                    // Pass
                                    logger.warn("Could not check for Phoenix SYSTEM tables, assuming they exist and are properly configured");
                                    checkClientServerCompatibility(SchemaUtil.getPhysicalName(SYSTEM_CATALOG_NAME_BYTES, getProps()).getName());
                                    success = true;
                                } else {
                                    initializationException = e;
                                }
                                return null;
                            }
                            if (!ConnectionQueryServicesImpl.this.upgradeRequired.get()) {
                                createOtherSystemTables(metaConnection);
                            } else if (isAutoUpgradeEnabled && !isDoNotUpgradePropSet) {
                                upgradeSystemTables(url, props);
                            }
                        }
                        scheduleRenewLeaseTasks();
                        success = true;
                    } catch (RetriableUpgradeException e) {
                        // to retry establishing connection.
                        throw e;
                    } catch (Exception e) {
                        if (e instanceof SQLException) {
                            initializationException = (SQLException) e;
                        } else {
                            // wrap every other exception into a SQLException
                            initializationException = new SQLException(e);
                        }
                    } finally {
                        try {
                            if (!success && hConnectionEstablished) {
                                connection.close();
                            }
                        } catch (IOException e) {
                            SQLException ex = new SQLException(e);
                            if (initializationException != null) {
                                initializationException.setNextException(ex);
                            } else {
                                initializationException = ex;
                            }
                        } finally {
                            try {
                                if (initializationException != null) {
                                    throw initializationException;
                                }
                            } finally {
                                initialized = true;
                            }
                        }
                    }
                }
                return null;
            }
        });
    } catch (Exception e) {
        Throwables.propagateIfInstanceOf(e, SQLException.class);
        Throwables.propagate(e);
    }
}

Example 59 with HBaseAdmin

use of org.apache.hadoop.hbase.client.HBaseAdmin in project phoenix by apache.

the class ConnectionQueryServicesImpl method restoreFromSnapshot.

private void restoreFromSnapshot(String tableName, String snapshotName, boolean success) throws SQLException {
    boolean snapshotRestored = false;
    boolean tableDisabled = false;
    if (!success && snapshotName != null) {
        SQLException sqlE = null;
        HBaseAdmin admin = null;
        try {
            logger.warn("Starting restore of " + tableName + " using snapshot " + snapshotName + " because upgrade failed");
            admin = getAdmin();
            admin.disableTable(tableName);
            tableDisabled = true;
            admin.restoreSnapshot(snapshotName);
            snapshotRestored = true;
            logger.warn("Successfully restored " + tableName + " using snapshot " + snapshotName);
        } catch (Exception e) {
            sqlE = new SQLException(e);
        } finally {
            if (admin != null && tableDisabled) {
                try {
                    admin.enableTable(tableName);
                    if (snapshotRestored) {
                        logger.warn("Successfully restored and enabled " + tableName + " using snapshot " + snapshotName);
                    } else {
                        logger.warn("Successfully enabled " + tableName + " after restoring using snapshot " + snapshotName + " failed. ");
                    }
                } catch (Exception e1) {
                    SQLException enableTableEx = new SQLException(e1);
                    if (sqlE == null) {
                        sqlE = enableTableEx;
                    } else {
                        sqlE.setNextException(enableTableEx);
                    }
                    logger.error("Failure in enabling " + tableName + (snapshotRestored ? " after successfully restoring using snapshot" + snapshotName : " after restoring using snapshot " + snapshotName + " failed. "));
                } finally {
                    try {
                        admin.close();
                    } catch (Exception e2) {
                        SQLException adminCloseEx = new SQLException(e2);
                        if (sqlE == null) {
                            sqlE = adminCloseEx;
                        } else {
                            sqlE.setNextException(adminCloseEx);
                        }
                    } finally {
                        if (sqlE != null) {
                            throw sqlE;
                        }
                    }
                }
            }
        }
    }
}

Example 60 with HBaseAdmin

use of org.apache.hadoop.hbase.client.HBaseAdmin in project phoenix by apache.

the class DropMetadataIT method testDropViewKeepsHTable.

@Test
public void testDropViewKeepsHTable() throws Exception {
    Connection conn = getConnection();
    HBaseAdmin admin = conn.unwrap(PhoenixConnection.class).getQueryServices().getAdmin();
    String hbaseNativeViewName = generateUniqueName();
    byte[] hbaseNativeBytes = SchemaUtil.getTableNameAsBytes(HBASE_NATIVE_SCHEMA_NAME, hbaseNativeViewName);
    try {
        @SuppressWarnings("deprecation") HTableDescriptor descriptor = new HTableDescriptor(hbaseNativeBytes);
        HColumnDescriptor columnDescriptor = new HColumnDescriptor(FAMILY_NAME);
        columnDescriptor.setKeepDeletedCells(true);
        descriptor.addFamily(columnDescriptor);
        admin.createTable(descriptor);
    } finally {
        admin.close();
    }
    conn.createStatement().execute("create view " + hbaseNativeViewName + "   (uint_key unsigned_int not null," + "    ulong_key unsigned_long not null," + "    string_key varchar not null,\n" + "    \"1\".uint_col unsigned_int," + "    \"1\".ulong_col unsigned_long" + "    CONSTRAINT pk PRIMARY KEY (uint_key, ulong_key, string_key))\n" + HColumnDescriptor.DATA_BLOCK_ENCODING + "='" + DataBlockEncoding.NONE + "'");
    conn.createStatement().execute("drop view " + hbaseNativeViewName);
    conn.close();
}