Examples with HBaseAdmin org.apache.hadoop.hbase.client.HBaseAdmin used on opensource projects

Example 66 with HBaseAdmin

use of org.apache.hadoop.hbase.client.HBaseAdmin in project phoenix by apache.

the class LocalIndexIT method testLocalIndexScanJoinColumnsFromDataTable.

@Test
public void testLocalIndexScanJoinColumnsFromDataTable() throws Exception {
    String tableName = schemaName + "." + generateUniqueName();
    String indexName = "IDX_" + generateUniqueName();
    String indexTableName = schemaName + "." + indexName;
    TableName physicalTableName = SchemaUtil.getPhysicalTableName(tableName.getBytes(), isNamespaceMapped);
    String indexPhysicalTableName = physicalTableName.getNameAsString();
    createBaseTable(tableName, null, "('e','i','o')");
    Connection conn1 = getConnection();
    try {
        conn1.createStatement().execute("UPSERT INTO " + tableName + " values('b',1,2,4,'z')");
        conn1.createStatement().execute("UPSERT INTO " + tableName + " values('f',1,2,3,'a')");
        conn1.createStatement().execute("UPSERT INTO " + tableName + " values('j',2,4,2,'a')");
        conn1.createStatement().execute("UPSERT INTO " + tableName + " values('q',3,1,1,'c')");
        conn1.commit();
        conn1.createStatement().execute("CREATE LOCAL INDEX " + indexName + " ON " + tableName + "(v1)");
        ResultSet rs = conn1.createStatement().executeQuery("SELECT COUNT(*) FROM " + indexTableName);
        assertTrue(rs.next());
        HBaseAdmin admin = driver.getConnectionQueryServices(getUrl(), TestUtil.TEST_PROPERTIES).getAdmin();
        int numRegions = admin.getTableRegions(physicalTableName).size();
        String query = "SELECT t_id, k1, k2, k3, V1 FROM " + tableName + " where v1='a'";
        rs = conn1.createStatement().executeQuery("EXPLAIN " + query);
        assertEquals("CLIENT PARALLEL " + numRegions + "-WAY RANGE SCAN OVER " + indexPhysicalTableName + " [1,'a']\n" + "    SERVER FILTER BY FIRST KEY ONLY\n" + "CLIENT MERGE SORT", QueryUtil.getExplainPlan(rs));
        rs = conn1.createStatement().executeQuery(query);
        assertTrue(rs.next());
        assertEquals("f", rs.getString("t_id"));
        assertEquals(1, rs.getInt("k1"));
        assertEquals(2, rs.getInt("k2"));
        assertEquals(3, rs.getInt("k3"));
        assertTrue(rs.next());
        assertEquals("j", rs.getString("t_id"));
        assertEquals(2, rs.getInt("k1"));
        assertEquals(4, rs.getInt("k2"));
        assertEquals(2, rs.getInt("k3"));
        assertFalse(rs.next());
        query = "SELECT t_id, k1, k2, k3, V1 from " + tableName + "  where v1<='z' order by V1,t_id";
        rs = conn1.createStatement().executeQuery("EXPLAIN " + query);
        assertEquals("CLIENT PARALLEL " + numRegions + "-WAY RANGE SCAN OVER " + indexPhysicalTableName + " [1,*] - [1,'z']\n" + "    SERVER FILTER BY FIRST KEY ONLY\n" + "CLIENT MERGE SORT", QueryUtil.getExplainPlan(rs));
        rs = conn1.createStatement().executeQuery(query);
        assertTrue(rs.next());
        assertEquals("f", rs.getString("t_id"));
        assertEquals(1, rs.getInt("k1"));
        assertEquals(2, rs.getInt("k2"));
        assertEquals(3, rs.getInt("k3"));
        assertEquals("a", rs.getString("V1"));
        assertTrue(rs.next());
        assertEquals("j", rs.getString("t_id"));
        assertEquals(2, rs.getInt("k1"));
        assertEquals(4, rs.getInt("k2"));
        assertEquals(2, rs.getInt("k3"));
        assertEquals("a", rs.getString("V1"));
        assertTrue(rs.next());
        assertEquals("q", rs.getString("t_id"));
        assertEquals(3, rs.getInt("k1"));
        assertEquals(1, rs.getInt("k2"));
        assertEquals(1, rs.getInt("k3"));
        assertEquals("c", rs.getString("V1"));
        assertTrue(rs.next());
        assertEquals("b", rs.getString("t_id"));
        assertEquals(1, rs.getInt("k1"));
        assertEquals(2, rs.getInt("k2"));
        assertEquals(4, rs.getInt("k3"));
        assertEquals("z", rs.getString("V1"));
        query = "SELECT t_id, V1, k3 from " + tableName + "  where v1 <='z' group by v1,t_id, k3";
        rs = conn1.createStatement().executeQuery("EXPLAIN " + query);
        assertEquals("CLIENT PARALLEL " + numRegions + "-WAY RANGE SCAN OVER " + indexPhysicalTableName + " [1,*] - [1,'z']\n" + "    SERVER FILTER BY FIRST KEY ONLY\n" + "    SERVER AGGREGATE INTO DISTINCT ROWS BY [\"V1\", \"T_ID\", \"K3\"]\nCLIENT MERGE SORT", QueryUtil.getExplainPlan(rs));
        rs = conn1.createStatement().executeQuery(query);
        assertTrue(rs.next());
        assertEquals("f", rs.getString("t_id"));
        assertEquals(3, rs.getInt("k3"));
        assertEquals("a", rs.getString("V1"));
        assertTrue(rs.next());
        assertEquals("j", rs.getString("t_id"));
        assertEquals(2, rs.getInt("k3"));
        assertEquals("a", rs.getString("V1"));
        assertTrue(rs.next());
        assertEquals("q", rs.getString("t_id"));
        assertEquals(1, rs.getInt("k3"));
        assertEquals("c", rs.getString("V1"));
        assertTrue(rs.next());
        assertEquals("b", rs.getString("t_id"));
        assertEquals(4, rs.getInt("k3"));
        assertEquals("z", rs.getString("V1"));
        query = "SELECT v1,sum(k3) from " + tableName + " where v1 <='z'  group by v1 order by v1";
        rs = conn1.createStatement().executeQuery("EXPLAIN " + query);
        assertEquals("CLIENT PARALLEL " + numRegions + "-WAY RANGE SCAN OVER " + indexPhysicalTableName + " [1,*] - [1,'z']\n" + "    SERVER FILTER BY FIRST KEY ONLY\n" + "    SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [\"V1\"]\nCLIENT MERGE SORT", QueryUtil.getExplainPlan(rs));
        PhoenixStatement stmt = conn1.createStatement().unwrap(PhoenixStatement.class);
        rs = stmt.executeQuery(query);
        QueryPlan plan = stmt.getQueryPlan();
        assertEquals(indexTableName, plan.getContext().getCurrentTable().getTable().getName().getString());
        assertEquals(BaseScannerRegionObserver.KEY_ORDERED_GROUP_BY_EXPRESSIONS, plan.getGroupBy().getScanAttribName());
        assertTrue(rs.next());
        assertEquals("a", rs.getString(1));
        assertEquals(5, rs.getInt(2));
        assertTrue(rs.next());
        assertEquals("c", rs.getString(1));
        assertEquals(1, rs.getInt(2));
        assertTrue(rs.next());
        assertEquals("z", rs.getString(1));
        assertEquals(4, rs.getInt(2));
    } finally {
        conn1.close();
    }
}

Example 67 with HBaseAdmin

use of org.apache.hadoop.hbase.client.HBaseAdmin in project phoenix by apache.

the class EndToEndCoveredIndexingIT method createSetupTables.

/**
     * Create the primary table (to which you should write), setup properly for indexing the given
     * {@link ColumnGroup}s. Also creates the necessary index tables to match the passes groups.
     * @param groups {@link ColumnGroup}s to index, creating one index table per column group.
     * @return reference to the primary table
     * @throws IOException if there is an issue communicating with HBase
     */
private HTable createSetupTables(ColumnGroup... groups) throws IOException {
    HBaseAdmin admin = UTIL.getHBaseAdmin();
    // setup the index
    CoveredColumnIndexSpecifierBuilder builder = new CoveredColumnIndexSpecifierBuilder();
    for (ColumnGroup group : groups) {
        builder.addIndexGroup(group);
        // create the index tables
        CoveredColumnIndexer.createIndexTable(admin, group.getTable());
    }
    // setup the primary table
    String indexedTableName = Bytes.toString(TestTable.getTableName());
    @SuppressWarnings("deprecation") HTableDescriptor pTable = new HTableDescriptor(indexedTableName);
    pTable.addFamily(new HColumnDescriptor(FAM));
    pTable.addFamily(new HColumnDescriptor(FAM2));
    builder.build(pTable);
    // create the primary table
    admin.createTable(pTable);
    HTable primary = new HTable(UTIL.getConfiguration(), indexedTableName);
    primary.setAutoFlush(false);
    return primary;
}

Example 68 with HBaseAdmin

use of org.apache.hadoop.hbase.client.HBaseAdmin in project phoenix by apache.

the class LocalIndexIT method testLocalIndexAutomaticRepair.

@Test
public void testLocalIndexAutomaticRepair() throws Exception {
    if (isNamespaceMapped) {
        return;
    }
    PhoenixConnection conn = DriverManager.getConnection(getUrl()).unwrap(PhoenixConnection.class);
    try (HTableInterface metaTable = conn.getQueryServices().getTable(TableName.META_TABLE_NAME.getName());
        HBaseAdmin admin = conn.getQueryServices().getAdmin()) {
        Statement statement = conn.createStatement();
        final String tableName = "T_AUTO_MATIC_REPAIR";
        String indexName = "IDX_T_AUTO_MATIC_REPAIR";
        String indexName1 = "IDX_T_AUTO_MATIC_REPAIR_1";
        statement.execute("create table " + tableName + " (id integer not null,fn varchar," + "cf1.ln varchar constraint pk primary key(id)) split on (1,2,3,4,5)");
        statement.execute("create local index " + indexName + " on " + tableName + "  (fn,cf1.ln)");
        statement.execute("create local index " + indexName1 + " on " + tableName + "  (fn)");
        for (int i = 0; i < 7; i++) {
            statement.execute("upsert into " + tableName + "  values(" + i + ",'fn" + i + "','ln" + i + "')");
        }
        conn.commit();
        ResultSet rs = statement.executeQuery("SELECT COUNT(*) FROM " + indexName);
        assertTrue(rs.next());
        assertEquals(7, rs.getLong(1));
        List<HRegionInfo> tableRegions = admin.getTableRegions(TableName.valueOf(tableName));
        admin.disableTable(tableName);
        copyLocalIndexHFiles(config, tableRegions.get(0), tableRegions.get(1), false);
        copyLocalIndexHFiles(config, tableRegions.get(3), tableRegions.get(0), false);
        admin.enableTable(tableName);
        int count = getCount(conn, tableName, "L#0");
        assertTrue(count > 14);
        admin.majorCompact(TableName.valueOf(tableName));
        // need to wait for rebuilding of corrupted local index region
        int tryCount = 5;
        while (tryCount-- > 0 && count != 14) {
            Thread.sleep(15000);
            count = getCount(conn, tableName, "L#0");
        }
        assertEquals(14, count);
        rs = statement.executeQuery("SELECT COUNT(*) FROM " + indexName1);
        assertTrue(rs.next());
        assertEquals(7, rs.getLong(1));
        statement.execute("DROP INDEX " + indexName1 + " ON " + tableName);
        admin.majorCompact(TableName.valueOf(tableName));
        statement.execute("DROP INDEX " + indexName + " ON " + tableName);
        admin.majorCompact(TableName.valueOf(tableName));
        Thread.sleep(15000);
        admin.majorCompact(TableName.valueOf(tableName));
        Thread.sleep(15000);
        rs = statement.executeQuery("SELECT COUNT(*) FROM " + tableName);
        assertTrue(rs.next());
    }
}

Example 69 with HBaseAdmin

use of org.apache.hadoop.hbase.client.HBaseAdmin in project phoenix by apache.

the class MutableIndexReplicationIT method testReplicationWithMutableIndexes.

@Test
public void testReplicationWithMutableIndexes() throws Exception {
    Connection conn = getConnection();
    //create the primary and index tables
    conn.createStatement().execute("CREATE TABLE " + DATA_TABLE_FULL_NAME + " (k VARCHAR NOT NULL PRIMARY KEY, v1 VARCHAR, v2 VARCHAR)");
    conn.createStatement().execute("CREATE INDEX " + INDEX_TABLE_NAME + " ON " + DATA_TABLE_FULL_NAME + " (v1)");
    // make sure that the tables are empty, but reachable
    String query = "SELECT * FROM " + DATA_TABLE_FULL_NAME;
    ResultSet rs = conn.createStatement().executeQuery(query);
    assertFalse(rs.next());
    //make sure there is no data in the table
    query = "SELECT * FROM " + INDEX_TABLE_FULL_NAME;
    rs = conn.createStatement().executeQuery(query);
    assertFalse(rs.next());
    // make sure the data tables are created on the remote cluster
    HBaseAdmin admin = utility1.getHBaseAdmin();
    HBaseAdmin admin2 = utility2.getHBaseAdmin();
    List<String> dataTables = new ArrayList<String>();
    dataTables.add(DATA_TABLE_FULL_NAME);
    dataTables.add(INDEX_TABLE_FULL_NAME);
    for (String tableName : dataTables) {
        HTableDescriptor desc = admin.getTableDescriptor(TableName.valueOf(tableName));
        //create it as-is on the remote cluster
        admin2.createTable(desc);
        LOG.info("Enabling replication on source table: " + tableName);
        HColumnDescriptor[] cols = desc.getColumnFamilies();
        assertEquals(1, cols.length);
        // add the replication scope to the column
        HColumnDescriptor col = desc.removeFamily(cols[0].getName());
        col.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
        desc.addFamily(col);
        //disable/modify/enable table so it has replication enabled
        admin.disableTable(desc.getTableName());
        admin.modifyTable(tableName, desc);
        admin.enableTable(desc.getTableName());
        LOG.info("Replication enabled on source table: " + tableName);
    }
    // load some data into the source cluster table
    PreparedStatement stmt = conn.prepareStatement("UPSERT INTO " + DATA_TABLE_FULL_NAME + " VALUES(?,?,?)");
    // k
    stmt.setString(1, "a");
    // v1 <- has index
    stmt.setString(2, "x");
    // v2
    stmt.setString(3, "1");
    stmt.execute();
    conn.commit();
    // make sure the index is working as expected
    query = "SELECT * FROM " + INDEX_TABLE_FULL_NAME;
    rs = conn.createStatement().executeQuery(query);
    assertTrue(rs.next());
    assertEquals("x", rs.getString(1));
    assertFalse(rs.next());
    conn.close();
    /*
         Validate that we have replicated the rows to the remote cluster
        */
    // other table can't be reached through Phoenix right now - would need to change how we
    // lookup tables. For right now, we just go through an HTable
    LOG.info("Looking up tables in replication target");
    TableName[] tables = admin2.listTableNames();
    HTable remoteTable = new HTable(utility2.getConfiguration(), tables[0]);
    for (int i = 0; i < REPLICATION_RETRIES; i++) {
        if (i >= REPLICATION_RETRIES - 1) {
            fail("Waited too much time for put replication on table " + remoteTable.getTableDescriptor().getNameAsString());
        }
        if (ensureAnyRows(remoteTable)) {
            break;
        }
        LOG.info("Sleeping for " + REPLICATION_WAIT_TIME_MILLIS + " for edits to get replicated");
        Thread.sleep(REPLICATION_WAIT_TIME_MILLIS);
    }
    remoteTable.close();
}

Example 70 with HBaseAdmin

use of org.apache.hadoop.hbase.client.HBaseAdmin in project phoenix by apache.

the class MetaDataClient method dropTable.

private MutationState dropTable(String schemaName, String tableName, String parentTableName, PTableType tableType, boolean ifExists, boolean cascade) throws SQLException {
    connection.rollback();
    boolean wasAutoCommit = connection.getAutoCommit();
    try {
        PName tenantId = connection.getTenantId();
        String tenantIdStr = tenantId == null ? null : tenantId.getString();
        byte[] key = SchemaUtil.getTableKey(tenantIdStr, schemaName, tableName);
        Long scn = connection.getSCN();
        long clientTimeStamp = scn == null ? HConstants.LATEST_TIMESTAMP : scn;
        List<Mutation> tableMetaData = Lists.newArrayListWithExpectedSize(2);
        Delete tableDelete = new Delete(key, clientTimeStamp);
        tableMetaData.add(tableDelete);
        boolean hasViewIndexTable = false;
        if (parentTableName != null) {
            byte[] linkKey = MetaDataUtil.getParentLinkKey(tenantIdStr, schemaName, parentTableName, tableName);
            Delete linkDelete = new Delete(linkKey, clientTimeStamp);
            tableMetaData.add(linkDelete);
        }
        MetaDataMutationResult result = connection.getQueryServices().dropTable(tableMetaData, tableType, cascade);
        MutationCode code = result.getMutationCode();
        PTable table = result.getTable();
        switch(code) {
            case TABLE_NOT_FOUND:
                if (!ifExists) {
                    throw new TableNotFoundException(schemaName, tableName);
                }
                break;
            case NEWER_TABLE_FOUND:
                throw new NewerTableAlreadyExistsException(schemaName, tableName, result.getTable());
            case UNALLOWED_TABLE_MUTATION:
                throw new SQLExceptionInfo.Builder(SQLExceptionCode.CANNOT_MUTATE_TABLE).setSchemaName(schemaName).setTableName(tableName).build().buildException();
            default:
                connection.removeTable(tenantId, SchemaUtil.getTableName(schemaName, tableName), parentTableName, result.getMutationTime());
                if (table != null) {
                    boolean dropMetaData = false;
                    long ts = (scn == null ? result.getMutationTime() : scn);
                    List<TableRef> tableRefs = Lists.newArrayListWithExpectedSize(2 + table.getIndexes().size());
                    connection.setAutoCommit(true);
                    if (tableType == PTableType.VIEW) {
                        for (PTable index : table.getIndexes()) {
                            tableRefs.add(new TableRef(null, index, ts, false));
                        }
                    } else {
                        dropMetaData = result.getTable().getViewIndexId() == null && connection.getQueryServices().getProps().getBoolean(DROP_METADATA_ATTRIB, DEFAULT_DROP_METADATA);
                        // All multi-tenant tables have a view index table, so no need to check in that case
                        if (parentTableName == null) {
                            // keeping always true for deletion of stats if view index present
                            hasViewIndexTable = true;
                            // or not
                            MetaDataUtil.deleteViewIndexSequences(connection, table.getPhysicalName(), table.isNamespaceMapped());
                            byte[] viewIndexPhysicalName = MetaDataUtil.getViewIndexPhysicalName(table.getPhysicalName().getBytes());
                            if (!dropMetaData) {
                                // we need to drop rows only when actually view index exists
                                try (HBaseAdmin admin = connection.getQueryServices().getAdmin()) {
                                    hasViewIndexTable = admin.tableExists(viewIndexPhysicalName);
                                } catch (IOException e1) {
                                // absorbing as it is not critical check
                                }
                            }
                        }
                        if (tableType == PTableType.TABLE && (table.isMultiTenant() || hasViewIndexTable)) {
                            if (hasViewIndexTable) {
                                byte[] viewIndexPhysicalName = MetaDataUtil.getViewIndexPhysicalName(table.getPhysicalName().getBytes());
                                PTable viewIndexTable = new PTableImpl(null, SchemaUtil.getSchemaNameFromFullName(viewIndexPhysicalName), SchemaUtil.getTableNameFromFullName(viewIndexPhysicalName), ts, table.getColumnFamilies(), table.isNamespaceMapped(), table.getImmutableStorageScheme(), table.getEncodingScheme(), table.useStatsForParallelization());
                                tableRefs.add(new TableRef(null, viewIndexTable, ts, false));
                            }
                        }
                        tableRefs.add(new TableRef(null, table, ts, false));
                        // TODO: Let the standard mutable secondary index maintenance handle this?
                        for (PTable index : table.getIndexes()) {
                            tableRefs.add(new TableRef(null, index, ts, false));
                        }
                        deleteFromStatsTable(tableRefs, ts);
                    }
                    if (!dropMetaData) {
                        MutationPlan plan = new PostDDLCompiler(connection).compile(tableRefs, null, null, Collections.<PColumn>emptyList(), ts);
                        // Delete everything in the column. You'll still be able to do queries at earlier timestamps
                        return connection.getQueryServices().updateData(plan);
                    }
                }
                break;
        }
        return new MutationState(0, 0, connection);
    } finally {
        connection.setAutoCommit(wasAutoCommit);
    }
}