Examples with HBaseAdmin org.apache.hadoop.hbase.client.HBaseAdmin used on opensource projects

Example 71 with HBaseAdmin

use of org.apache.hadoop.hbase.client.HBaseAdmin in project phoenix by apache.

the class UpgradeIT method testMapTableToNamespaceDuringUpgrade.

@Test
public void testMapTableToNamespaceDuringUpgrade() throws SQLException, IOException, IllegalArgumentException, InterruptedException {
    String[] strings = new String[] { "a", "b", "c", "d" };
    try (Connection conn = DriverManager.getConnection(getUrl())) {
        String schemaName = "TEST";
        String phoenixFullTableName = schemaName + "." + generateUniqueName();
        String indexName = "IDX_" + generateUniqueName();
        String localIndexName = "LIDX_" + generateUniqueName();
        String viewName = "VIEW_" + generateUniqueName();
        String viewIndexName = "VIDX_" + generateUniqueName();
        String[] tableNames = new String[] { phoenixFullTableName, schemaName + "." + indexName, schemaName + "." + localIndexName, "diff." + viewName, "test." + viewName, viewName };
        String[] viewIndexes = new String[] { "diff." + viewIndexName, "test." + viewIndexName };
        conn.createStatement().execute("CREATE TABLE " + phoenixFullTableName + "(k VARCHAR PRIMARY KEY, v INTEGER, f INTEGER, g INTEGER NULL, h INTEGER NULL)");
        PreparedStatement upsertStmt = conn.prepareStatement("UPSERT INTO " + phoenixFullTableName + " VALUES(?, ?, 0, 0, 0)");
        int i = 1;
        for (String str : strings) {
            upsertStmt.setString(1, str);
            upsertStmt.setInt(2, i++);
            upsertStmt.execute();
        }
        conn.commit();
        // creating local index
        conn.createStatement().execute("create local index " + localIndexName + " on " + phoenixFullTableName + "(K)");
        // creating global index
        conn.createStatement().execute("create index " + indexName + " on " + phoenixFullTableName + "(k)");
        // creating view in schema 'diff'
        conn.createStatement().execute("CREATE VIEW diff." + viewName + " (col VARCHAR) AS SELECT * FROM " + phoenixFullTableName);
        // creating view in schema 'test'
        conn.createStatement().execute("CREATE VIEW test." + viewName + " (col VARCHAR) AS SELECT * FROM " + phoenixFullTableName);
        conn.createStatement().execute("CREATE VIEW " + viewName + "(col VARCHAR) AS SELECT * FROM " + phoenixFullTableName);
        // Creating index on views
        conn.createStatement().execute("create index " + viewIndexName + "  on diff." + viewName + "(col)");
        conn.createStatement().execute("create index " + viewIndexName + " on test." + viewName + "(col)");
        // validate data
        for (String tableName : tableNames) {
            ResultSet rs = conn.createStatement().executeQuery("select * from " + tableName);
            for (String str : strings) {
                assertTrue(rs.next());
                assertEquals(str, rs.getString(1));
            }
        }
        // validate view Index data
        for (String viewIndex : viewIndexes) {
            ResultSet rs = conn.createStatement().executeQuery("select * from " + viewIndex);
            for (String str : strings) {
                assertTrue(rs.next());
                assertEquals(str, rs.getString(2));
            }
        }
        HBaseAdmin admin = conn.unwrap(PhoenixConnection.class).getQueryServices().getAdmin();
        assertTrue(admin.tableExists(phoenixFullTableName));
        assertTrue(admin.tableExists(schemaName + QueryConstants.NAME_SEPARATOR + indexName));
        assertTrue(admin.tableExists(MetaDataUtil.getViewIndexPhysicalName(Bytes.toBytes(phoenixFullTableName))));
        Properties props = new Properties();
        props.setProperty(QueryServices.IS_NAMESPACE_MAPPING_ENABLED, Boolean.toString(true));
        props.setProperty(QueryServices.IS_SYSTEM_TABLE_MAPPED_TO_NAMESPACE, Boolean.toString(false));
        admin.close();
        PhoenixConnection phxConn = DriverManager.getConnection(getUrl(), props).unwrap(PhoenixConnection.class);
        UpgradeUtil.upgradeTable(phxConn, phoenixFullTableName);
        UpgradeUtil.mapChildViewsToNamespace(phxConn, phoenixFullTableName, props);
        phxConn.close();
        props = new Properties();
        phxConn = DriverManager.getConnection(getUrl(), props).unwrap(PhoenixConnection.class);
        admin = phxConn.getQueryServices().getAdmin();
        String hbaseTableName = SchemaUtil.getPhysicalTableName(Bytes.toBytes(phoenixFullTableName), true).getNameAsString();
        assertTrue(admin.tableExists(hbaseTableName));
        assertTrue(admin.tableExists(Bytes.toBytes(hbaseTableName)));
        assertTrue(admin.tableExists(schemaName + QueryConstants.NAMESPACE_SEPARATOR + indexName));
        assertTrue(admin.tableExists(MetaDataUtil.getViewIndexPhysicalName(Bytes.toBytes(hbaseTableName))));
        i = 0;
        // validate data
        for (String tableName : tableNames) {
            ResultSet rs = phxConn.createStatement().executeQuery("select * from " + tableName);
            for (String str : strings) {
                assertTrue(rs.next());
                assertEquals(str, rs.getString(1));
            }
        }
        // validate view Index data
        for (String viewIndex : viewIndexes) {
            ResultSet rs = conn.createStatement().executeQuery("select * from " + viewIndex);
            for (String str : strings) {
                assertTrue(rs.next());
                assertEquals(str, rs.getString(2));
            }
        }
        PName tenantId = phxConn.getTenantId();
        PName physicalName = PNameFactory.newName(hbaseTableName);
        String oldSchemaName = MetaDataUtil.getViewIndexSequenceSchemaName(PNameFactory.newName(phoenixFullTableName), false);
        String newSchemaName = MetaDataUtil.getViewIndexSequenceSchemaName(physicalName, true);
        String newSequenceName = MetaDataUtil.getViewIndexSequenceName(physicalName, tenantId, true);
        ResultSet rs = phxConn.createStatement().executeQuery("SELECT " + PhoenixDatabaseMetaData.CURRENT_VALUE + "  FROM " + PhoenixDatabaseMetaData.SYSTEM_SEQUENCE + " WHERE " + PhoenixDatabaseMetaData.TENANT_ID + " IS NULL AND " + PhoenixDatabaseMetaData.SEQUENCE_SCHEMA + " = '" + newSchemaName + "' AND " + PhoenixDatabaseMetaData.SEQUENCE_NAME + "='" + newSequenceName + "'");
        assertTrue(rs.next());
        assertEquals("-32765", rs.getString(1));
        rs = phxConn.createStatement().executeQuery("SELECT " + PhoenixDatabaseMetaData.SEQUENCE_SCHEMA + "," + PhoenixDatabaseMetaData.SEQUENCE_SCHEMA + "," + PhoenixDatabaseMetaData.CURRENT_VALUE + "  FROM " + PhoenixDatabaseMetaData.SYSTEM_SEQUENCE + " WHERE " + PhoenixDatabaseMetaData.TENANT_ID + " IS NULL AND " + PhoenixDatabaseMetaData.SEQUENCE_SCHEMA + " = '" + oldSchemaName + "'");
        assertFalse(rs.next());
        phxConn.close();
        admin.close();
    }
}

Example 72 with HBaseAdmin

use of org.apache.hadoop.hbase.client.HBaseAdmin in project phoenix by apache.

the class UseSchemaIT method testMappedView.

@Test
public void testMappedView() throws Exception {
    Properties props = new Properties();
    String schema = generateUniqueName();
    String tableName = generateUniqueName();
    String fullTablename = schema + QueryConstants.NAME_SEPARATOR + tableName;
    props.setProperty(QueryServices.SCHEMA_ATTRIB, schema);
    Connection conn = DriverManager.getConnection(getUrl(), props);
    HBaseAdmin admin = driver.getConnectionQueryServices(getUrl(), TestUtil.TEST_PROPERTIES).getAdmin();
    admin.createNamespace(NamespaceDescriptor.create(schema).build());
    admin.createTable(new HTableDescriptor(fullTablename).addFamily(new HColumnDescriptor(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES)));
    Put put = new Put(PVarchar.INSTANCE.toBytes(fullTablename));
    put.addColumn(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, QueryConstants.EMPTY_COLUMN_BYTES, QueryConstants.EMPTY_COLUMN_VALUE_BYTES);
    HTable phoenixSchematable = new HTable(admin.getConfiguration(), fullTablename);
    phoenixSchematable.put(put);
    phoenixSchematable.close();
    conn.createStatement().execute("CREATE VIEW " + tableName + " (tablename VARCHAR PRIMARY KEY)");
    ResultSet rs = conn.createStatement().executeQuery("select tablename from " + tableName);
    assertTrue(rs.next());
    assertEquals(fullTablename, rs.getString(1));
    admin.close();
    conn.close();
}

Example 73 with HBaseAdmin

use of org.apache.hadoop.hbase.client.HBaseAdmin in project phoenix by apache.

the class ViewIT method testViewAndTableInDifferentSchemas.

public void testViewAndTableInDifferentSchemas(boolean isNamespaceMapped) throws Exception {
    Properties props = new Properties();
    props.setProperty(QueryServices.IS_NAMESPACE_MAPPING_ENABLED, Boolean.toString(isNamespaceMapped));
    Connection conn = DriverManager.getConnection(getUrl(), props);
    String schemaName1 = "S_" + generateUniqueName();
    String fullTableName1 = SchemaUtil.getTableName(schemaName1, tableName);
    if (isNamespaceMapped) {
        conn.createStatement().execute("CREATE SCHEMA IF NOT EXISTS " + schemaName1);
    }
    String ddl = "CREATE TABLE " + fullTableName1 + " (k INTEGER NOT NULL PRIMARY KEY, v1 DATE)" + tableDDLOptions;
    HBaseAdmin admin = conn.unwrap(PhoenixConnection.class).getQueryServices().getAdmin();
    conn.createStatement().execute(ddl);
    assertTrue(admin.tableExists(SchemaUtil.getPhysicalTableName(SchemaUtil.normalizeIdentifier(fullTableName1), conn.unwrap(PhoenixConnection.class).getQueryServices().getProps())));
    String viewName = "V_" + generateUniqueName();
    String viewSchemaName = "S_" + generateUniqueName();
    String fullViewName1 = SchemaUtil.getTableName(viewSchemaName, viewName);
    ddl = "CREATE VIEW " + fullViewName1 + " (v2 VARCHAR) AS SELECT * FROM " + fullTableName1 + " WHERE k > 5";
    conn.createStatement().execute(ddl);
    String fullViewName2 = "V_" + generateUniqueName();
    ddl = "CREATE VIEW " + fullViewName2 + " (v2 VARCHAR) AS SELECT * FROM " + fullTableName1 + " WHERE k > 5";
    conn.createStatement().execute(ddl);
    conn.createStatement().executeQuery("SELECT * FROM " + fullViewName1);
    conn.createStatement().executeQuery("SELECT * FROM " + fullViewName2);
    ddl = "DROP VIEW " + viewName;
    try {
        conn.createStatement().execute(ddl);
        fail();
    } catch (TableNotFoundException ignore) {
    }
    ddl = "DROP VIEW " + fullViewName1;
    conn.createStatement().execute(ddl);
    ddl = "DROP VIEW " + SchemaUtil.getTableName(viewSchemaName, generateUniqueName());
    try {
        conn.createStatement().execute(ddl);
        fail();
    } catch (TableNotFoundException ignore) {
    }
    ddl = "DROP TABLE " + fullTableName1;
    validateCannotDropTableWithChildViewsWithoutCascade(conn, fullTableName1);
    ddl = "DROP VIEW " + fullViewName2;
    conn.createStatement().execute(ddl);
    ddl = "DROP TABLE " + fullTableName1;
    conn.createStatement().execute(ddl);
}

Example 74 with HBaseAdmin

use of org.apache.hadoop.hbase.client.HBaseAdmin in project phoenix by apache.

the class UpgradeUtil method disableViewIndexes.

public static PhoenixConnection disableViewIndexes(PhoenixConnection connParam) throws SQLException, IOException, InterruptedException, TimeoutException {
    Properties props = PropertiesUtil.deepCopy(connParam.getClientInfo());
    Long originalScn = null;
    String str = props.getProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB);
    if (str != null) {
        originalScn = Long.valueOf(str);
    }
    // don't use the passed timestamp as scn because we want to query all view indexes up to now.
    props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(HConstants.LATEST_TIMESTAMP));
    Set<String> physicalTables = new HashSet<>();
    SQLException sqlEx = null;
    PhoenixConnection globalConnection = null;
    PhoenixConnection toReturn = null;
    try {
        globalConnection = new PhoenixConnection(connParam, connParam.getQueryServices(), props);
        String tenantId = null;
        try (HBaseAdmin admin = globalConnection.getQueryServices().getAdmin()) {
            String fetchViewIndexes = "SELECT " + TENANT_ID + ", " + TABLE_SCHEM + ", " + TABLE_NAME + ", " + DATA_TABLE_NAME + " FROM " + SYSTEM_CATALOG_NAME + " WHERE " + VIEW_INDEX_ID + " IS NOT NULL";
            String disableIndexDDL = "ALTER INDEX %s ON %s DISABLE";
            try (ResultSet rs = globalConnection.createStatement().executeQuery(fetchViewIndexes)) {
                while (rs.next()) {
                    tenantId = rs.getString(1);
                    String indexSchema = rs.getString(2);
                    String indexName = rs.getString(3);
                    String viewName = rs.getString(4);
                    String fullIndexName = SchemaUtil.getTableName(indexSchema, indexName);
                    String fullViewName = SchemaUtil.getTableName(indexSchema, viewName);
                    PTable viewPTable = null;
                    // Users would need to rebuild the view indexes. 
                    if (tenantId != null && !tenantId.isEmpty()) {
                        Properties newProps = PropertiesUtil.deepCopy(globalConnection.getClientInfo());
                        newProps.setProperty(PhoenixRuntime.TENANT_ID_ATTRIB, tenantId);
                        PTable indexPTable = null;
                        try (PhoenixConnection tenantConnection = new PhoenixConnection(globalConnection, globalConnection.getQueryServices(), newProps)) {
                            viewPTable = PhoenixRuntime.getTable(tenantConnection, fullViewName);
                            tenantConnection.createStatement().execute(String.format(disableIndexDDL, indexName, fullViewName));
                            indexPTable = PhoenixRuntime.getTable(tenantConnection, fullIndexName);
                        }
                        int offset = indexPTable.getBucketNum() != null ? 1 : 0;
                        // positions are stored 1 based
                        int existingTenantIdPosition = ++offset;
                        int existingViewIdxIdPosition = ++offset;
                        int newTenantIdPosition = existingViewIdxIdPosition;
                        int newViewIdxPosition = existingTenantIdPosition;
                        String tenantIdColumn = indexPTable.getColumns().get(existingTenantIdPosition - 1).getName().getString();
                        int index = 0;
                        String updatePosition = "UPSERT INTO " + SYSTEM_CATALOG_NAME + " ( " + TENANT_ID + "," + TABLE_SCHEM + "," + TABLE_NAME + "," + COLUMN_NAME + "," + COLUMN_FAMILY + "," + ORDINAL_POSITION + ") SELECT " + TENANT_ID + "," + TABLE_SCHEM + "," + TABLE_NAME + "," + COLUMN_NAME + "," + COLUMN_FAMILY + "," + "?" + " FROM " + SYSTEM_CATALOG_NAME + " WHERE " + TENANT_ID + " = ? " + " AND " + TABLE_NAME + " = ? " + " AND " + (indexSchema == null ? TABLE_SCHEM + " IS NULL" : TABLE_SCHEM + " = ? ") + " AND " + COLUMN_NAME + " = ? ";
                        // update view index position
                        try (PreparedStatement s = globalConnection.prepareStatement(updatePosition)) {
                            index = 0;
                            s.setInt(++index, newViewIdxPosition);
                            s.setString(++index, tenantId);
                            s.setString(++index, indexName);
                            if (indexSchema != null) {
                                s.setString(++index, indexSchema);
                            }
                            s.setString(++index, MetaDataUtil.getViewIndexIdColumnName());
                            s.executeUpdate();
                        }
                        // update tenant id position
                        try (PreparedStatement s = globalConnection.prepareStatement(updatePosition)) {
                            index = 0;
                            s.setInt(++index, newTenantIdPosition);
                            s.setString(++index, tenantId);
                            s.setString(++index, indexName);
                            if (indexSchema != null) {
                                s.setString(++index, indexSchema);
                            }
                            s.setString(++index, tenantIdColumn);
                            s.executeUpdate();
                        }
                        globalConnection.commit();
                    } else {
                        viewPTable = PhoenixRuntime.getTable(globalConnection, fullViewName);
                        globalConnection.createStatement().execute(String.format(disableIndexDDL, indexName, fullViewName));
                    }
                    String indexPhysicalTableName = MetaDataUtil.getViewIndexTableName(viewPTable.getPhysicalName().getString());
                    if (physicalTables.add(indexPhysicalTableName)) {
                        final TableName tableName = TableName.valueOf(indexPhysicalTableName);
                        admin.disableTable(tableName);
                        admin.truncateTable(tableName, false);
                    }
                }
            }
        }
        if (originalScn != null) {
            props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(originalScn));
        }
        toReturn = new PhoenixConnection(globalConnection, globalConnection.getQueryServices(), props);
    } catch (SQLException e) {
        sqlEx = e;
    } finally {
        sqlEx = closeConnection(connParam, sqlEx);
        sqlEx = closeConnection(globalConnection, sqlEx);
        if (sqlEx != null) {
            throw sqlEx;
        }
    }
    return toReturn;
}

Example 75 with HBaseAdmin

use of org.apache.hadoop.hbase.client.HBaseAdmin in project phoenix by apache.

the class UpgradeUtil method upgradeDescVarLengthRowKeys.

private static void upgradeDescVarLengthRowKeys(PhoenixConnection upgradeConn, PhoenixConnection globalConn, String schemaName, String tableName, boolean isTable, boolean bypassUpgrade) throws SQLException {
    String physicalName = SchemaUtil.getTableName(schemaName, tableName);
    long currentTime = System.currentTimeMillis();
    String snapshotName = physicalName + "_" + currentTime;
    HBaseAdmin admin = null;
    if (isTable && !bypassUpgrade) {
        admin = globalConn.getQueryServices().getAdmin();
    }
    boolean restoreSnapshot = false;
    boolean success = false;
    try {
        if (isTable && !bypassUpgrade) {
            String msg = "Taking snapshot of physical table " + physicalName + " prior to upgrade...";
            System.out.println(msg);
            logger.info(msg);
            admin.disableTable(physicalName);
            admin.snapshot(snapshotName, physicalName);
            admin.enableTable(physicalName);
            restoreSnapshot = true;
        }
        String escapedTableName = SchemaUtil.getEscapedTableName(schemaName, tableName);
        String tenantInfo = "";
        PName tenantId = PName.EMPTY_NAME;
        if (upgradeConn.getTenantId() != null) {
            tenantId = upgradeConn.getTenantId();
            tenantInfo = " for tenant " + tenantId.getString();
        }
        String msg = "Starting upgrade of " + escapedTableName + tenantInfo + "...";
        System.out.println(msg);
        logger.info(msg);
        ResultSet rs;
        if (!bypassUpgrade) {
            rs = upgradeConn.createStatement().executeQuery("SELECT /*+ NO_INDEX */ count(*) FROM " + escapedTableName);
            // Run query
            rs.next();
        }
        List<String> tableNames = Lists.newArrayListWithExpectedSize(1024);
        tableNames.add(tenantId == PName.EMPTY_NAME ? null : tenantId.getString());
        tableNames.add(schemaName);
        tableNames.add(tableName);
        // Find views to mark as upgraded
        if (isTable) {
            String query = "SELECT TENANT_ID,TABLE_SCHEM,TABLE_NAME\n" + "FROM SYSTEM.CATALOG\n" + "WHERE COLUMN_NAME IS NULL\n" + "AND COLUMN_FAMILY = '" + physicalName + "'" + "AND LINK_TYPE = " + LinkType.PHYSICAL_TABLE.getSerializedValue();
            rs = globalConn.createStatement().executeQuery(query);
            while (rs.next()) {
                tableNames.add(rs.getString(1));
                tableNames.add(rs.getString(2));
                tableNames.add(rs.getString(3));
            }
        }
        // Mark the table and views as upgraded now
        for (int i = 0; i < tableNames.size(); i += 3) {
            String theTenantId = tableNames.get(i);
            String theSchemaName = tableNames.get(i + 1);
            String theTableName = tableNames.get(i + 2);
            globalConn.createStatement().execute("UPSERT INTO " + PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME + " (" + PhoenixDatabaseMetaData.TENANT_ID + "," + PhoenixDatabaseMetaData.TABLE_SCHEM + "," + PhoenixDatabaseMetaData.TABLE_NAME + "," + MetaDataEndpointImpl.ROW_KEY_ORDER_OPTIMIZABLE + " BOOLEAN" + ") VALUES (" + "'" + (theTenantId == null ? StringUtil.EMPTY_STRING : theTenantId) + "'," + "'" + (theSchemaName == null ? StringUtil.EMPTY_STRING : theSchemaName) + "'," + "'" + theTableName + "'," + "TRUE)");
        }
        globalConn.commit();
        for (int i = 0; i < tableNames.size(); i += 3) {
            String theTenantId = tableNames.get(i);
            String theSchemaName = tableNames.get(i + 1);
            String theTableName = tableNames.get(i + 2);
            globalConn.getQueryServices().clearTableFromCache(theTenantId == null ? ByteUtil.EMPTY_BYTE_ARRAY : Bytes.toBytes(theTenantId), theSchemaName == null ? ByteUtil.EMPTY_BYTE_ARRAY : Bytes.toBytes(schemaName), Bytes.toBytes(theTableName), HConstants.LATEST_TIMESTAMP);
        }
        success = true;
        msg = "Completed upgrade of " + escapedTableName + tenantInfo;
        System.out.println(msg);
        logger.info(msg);
    } catch (Exception e) {
        logger.error("Exception during upgrade of " + physicalName + ":", e);
    } finally {
        boolean restored = false;
        try {
            if (!success && restoreSnapshot) {
                admin.disableTable(physicalName);
                admin.restoreSnapshot(snapshotName, false);
                admin.enableTable(physicalName);
                String msg = "Restored snapshot of " + physicalName + " due to failure of upgrade";
                System.out.println(msg);
                logger.info(msg);
            }
            restored = true;
        } catch (Exception e) {
            logger.warn("Unable to restoring snapshot " + snapshotName + " after failed upgrade", e);
        } finally {
            try {
                if (restoreSnapshot && restored) {
                    admin.deleteSnapshot(snapshotName);
                }
            } catch (Exception e) {
                logger.warn("Unable to delete snapshot " + snapshotName + " after upgrade:", e);
            } finally {
                try {
                    if (admin != null) {
                        admin.close();
                    }
                } catch (IOException e) {
                    logger.warn("Unable to close admin after upgrade:", e);
                }
            }
        }
    }
}