Examples with PTable org.apache.phoenix.schema.PTable used on opensource projects

Example 31 with PTable

use of org.apache.phoenix.schema.PTable in project phoenix by apache.

the class IndexTool method run.

@Override
public int run(String[] args) throws Exception {
    Connection connection = null;
    HTable htable = null;
    try {
        CommandLine cmdLine = null;
        try {
            cmdLine = parseOptions(args);
        } catch (IllegalStateException e) {
            printHelpAndExit(e.getMessage(), getOptions());
        }
        final Configuration configuration = HBaseConfiguration.addHbaseResources(getConf());
        final String schemaName = cmdLine.getOptionValue(SCHEMA_NAME_OPTION.getOpt());
        final String dataTable = cmdLine.getOptionValue(DATA_TABLE_OPTION.getOpt());
        final String indexTable = cmdLine.getOptionValue(INDEX_TABLE_OPTION.getOpt());
        final boolean isPartialBuild = cmdLine.hasOption(PARTIAL_REBUILD_OPTION.getOpt());
        final String qDataTable = SchemaUtil.getQualifiedTableName(schemaName, dataTable);
        boolean useDirectApi = cmdLine.hasOption(DIRECT_API_OPTION.getOpt());
        String basePath = cmdLine.getOptionValue(OUTPUT_PATH_OPTION.getOpt());
        boolean isForeground = cmdLine.hasOption(RUN_FOREGROUND_OPTION.getOpt());
        connection = ConnectionUtil.getInputConnection(configuration);
        byte[][] splitKeysBeforeJob = null;
        boolean isLocalIndexBuild = false;
        PTable pindexTable = null;
        if (indexTable != null) {
            if (!isValidIndexTable(connection, qDataTable, indexTable)) {
                throw new IllegalArgumentException(String.format(" %s is not an index table for %s ", indexTable, qDataTable));
            }
            pindexTable = PhoenixRuntime.getTable(connection, schemaName != null && !schemaName.isEmpty() ? SchemaUtil.getQualifiedTableName(schemaName, indexTable) : indexTable);
            htable = (HTable) connection.unwrap(PhoenixConnection.class).getQueryServices().getTable(pindexTable.getPhysicalName().getBytes());
            if (IndexType.LOCAL.equals(pindexTable.getIndexType())) {
                isLocalIndexBuild = true;
                splitKeysBeforeJob = htable.getRegionLocator().getStartKeys();
            }
        }
        PTable pdataTable = PhoenixRuntime.getTableNoCache(connection, qDataTable);
        Path outputPath = null;
        FileSystem fs = null;
        if (basePath != null) {
            outputPath = CsvBulkImportUtil.getOutputPath(new Path(basePath), pindexTable == null ? pdataTable.getPhysicalName().getString() : pindexTable.getPhysicalName().getString());
            fs = outputPath.getFileSystem(configuration);
            fs.delete(outputPath, true);
        }
        Job job = new JobFactory(connection, configuration, outputPath).getJob(schemaName, indexTable, dataTable, useDirectApi, isPartialBuild);
        if (!isForeground && useDirectApi) {
            LOG.info("Running Index Build in Background - Submit async and exit");
            job.submit();
            return 0;
        }
        LOG.info("Running Index Build in Foreground. Waits for the build to complete. This may take a long time!.");
        boolean result = job.waitForCompletion(true);
        if (result) {
            if (!useDirectApi && indexTable != null) {
                if (isLocalIndexBuild) {
                    validateSplitForLocalIndex(splitKeysBeforeJob, htable);
                }
                LOG.info("Loading HFiles from {}", outputPath);
                LoadIncrementalHFiles loader = new LoadIncrementalHFiles(configuration);
                loader.doBulkLoad(outputPath, htable);
                htable.close();
                // Without direct API, we need to update the index state to ACTIVE from client.
                IndexToolUtil.updateIndexState(connection, qDataTable, indexTable, PIndexState.ACTIVE);
                fs.delete(outputPath, true);
            }
            return 0;
        } else {
            LOG.error("IndexTool job failed! Check logs for errors..");
            return -1;
        }
    } catch (Exception ex) {
        LOG.error("An exception occurred while performing the indexing job: " + ExceptionUtils.getMessage(ex) + " at:\n" + ExceptionUtils.getStackTrace(ex));
        return -1;
    } finally {
        try {
            if (connection != null) {
                connection.close();
            }
            if (htable != null) {
                htable.close();
            }
        } catch (SQLException sqle) {
            LOG.error("Failed to close connection ", sqle.getMessage());
            throw new RuntimeException("Failed to close connection");
        }
    }
}

Example 32 with PTable

use of org.apache.phoenix.schema.PTable in project phoenix by apache.

the class QueryOptimizer method getHintedQueryPlan.

private static QueryPlan getHintedQueryPlan(PhoenixStatement statement, SelectStatement select, List<PTable> indexes, List<? extends PDatum> targetColumns, ParallelIteratorFactory parallelIteratorFactory, List<QueryPlan> plans) throws SQLException {
    QueryPlan dataPlan = plans.get(0);
    String indexHint = select.getHint().getHint(Hint.INDEX);
    if (indexHint == null) {
        return null;
    }
    int startIndex = 0;
    String alias = dataPlan.getTableRef().getTableAlias();
    String prefix = HintNode.PREFIX + (alias == null ? dataPlan.getTableRef().getTable().getName().getString() : alias) + HintNode.SEPARATOR;
    while (startIndex < indexHint.length()) {
        startIndex = indexHint.indexOf(prefix, startIndex);
        if (startIndex < 0) {
            return null;
        }
        startIndex += prefix.length();
        // true when SUFFIX found
        boolean done = false;
        while (startIndex < indexHint.length() && !done) {
            int endIndex;
            int endIndex1 = indexHint.indexOf(HintNode.SEPARATOR, startIndex);
            int endIndex2 = indexHint.indexOf(HintNode.SUFFIX, startIndex);
            if (endIndex1 < 0 && endIndex2 < 0) {
                // Missing SUFFIX shouldn't happen
                endIndex = indexHint.length();
            } else if (endIndex1 < 0) {
                done = true;
                endIndex = endIndex2;
            } else if (endIndex2 < 0) {
                endIndex = endIndex1;
            } else {
                endIndex = Math.min(endIndex1, endIndex2);
                done = endIndex2 == endIndex;
            }
            String indexName = indexHint.substring(startIndex, endIndex);
            int indexPos = getIndexPosition(indexes, indexName);
            if (indexPos >= 0) {
                // Hinted index is applicable, so return it's index
                PTable index = indexes.get(indexPos);
                indexes.remove(indexPos);
                QueryPlan plan = addPlan(statement, select, index, targetColumns, parallelIteratorFactory, dataPlan, true);
                if (plan != null) {
                    return plan;
                }
            }
            startIndex = endIndex + 1;
        }
    }
    return null;
}

Example 33 with PTable

use of org.apache.phoenix.schema.PTable in project phoenix by apache.

the class UpgradeUtil method upgradeDescVarLengthRowKeys.

/**
     * Upgrade tables and their indexes due to a bug causing descending row keys to have a row key that
     * prevents them from being sorted correctly (PHOENIX-2067).
     */
public static void upgradeDescVarLengthRowKeys(PhoenixConnection conn, List<String> tablesToUpgrade, boolean bypassUpgrade) throws SQLException {
    if (tablesToUpgrade.isEmpty()) {
        return;
    }
    List<PTable> tablesNeedingUpgrading = Lists.newArrayListWithExpectedSize(tablesToUpgrade.size());
    List<String> invalidTables = Lists.newArrayListWithExpectedSize(tablesToUpgrade.size());
    for (String fullTableName : tablesToUpgrade) {
        PTable table = PhoenixRuntime.getTable(conn, fullTableName);
        if (isInvalidTableToUpgrade(table)) {
            invalidTables.add(fullTableName);
        } else {
            tablesNeedingUpgrading.add(table);
        }
    }
    if (!invalidTables.isEmpty()) {
        StringBuilder buf = new StringBuilder("Only physical tables should be upgraded as their views and indexes will be updated with them: ");
        for (String fullTableName : invalidTables) {
            buf.append(fullTableName);
            buf.append(' ');
        }
        throw new SQLException(buf.toString());
    }
    PhoenixConnection upgradeConn = new PhoenixConnection(conn, true, true);
    try {
        upgradeConn.setAutoCommit(true);
        for (PTable table : tablesNeedingUpgrading) {
            boolean wasUpgraded = false;
            if (!table.rowKeyOrderOptimizable()) {
                wasUpgraded = true;
                upgradeDescVarLengthRowKeys(upgradeConn, conn, table.getSchemaName().getString(), table.getTableName().getString(), true, bypassUpgrade);
            }
            // Upgrade global indexes
            for (PTable index : table.getIndexes()) {
                if (!index.rowKeyOrderOptimizable() && index.getIndexType() != IndexType.LOCAL) {
                    wasUpgraded = true;
                    upgradeDescVarLengthRowKeys(upgradeConn, conn, index.getSchemaName().getString(), index.getTableName().getString(), false, bypassUpgrade);
                }
            }
            String sharedViewIndexName = Bytes.toString(MetaDataUtil.getViewIndexPhysicalName(table.getName().getBytes()));
            // Upgrade view indexes
            wasUpgraded |= upgradeSharedIndex(upgradeConn, conn, sharedViewIndexName, bypassUpgrade);
            String sharedLocalIndexName = Bytes.toString(MetaDataUtil.getLocalIndexPhysicalName(table.getName().getBytes()));
            // Upgrade local indexes
            wasUpgraded |= upgradeSharedIndex(upgradeConn, conn, sharedLocalIndexName, bypassUpgrade);
            if (!wasUpgraded) {
                System.out.println("Upgrade not required for this table or its indexes: " + table.getName().getString());
            }
        }
    } finally {
        upgradeConn.close();
    }
}

Example 34 with PTable

use of org.apache.phoenix.schema.PTable in project phoenix by apache.

the class UpgradeUtil method disableViewIndexes.

public static PhoenixConnection disableViewIndexes(PhoenixConnection connParam) throws SQLException, IOException, InterruptedException, TimeoutException {
    Properties props = PropertiesUtil.deepCopy(connParam.getClientInfo());
    Long originalScn = null;
    String str = props.getProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB);
    if (str != null) {
        originalScn = Long.valueOf(str);
    }
    // don't use the passed timestamp as scn because we want to query all view indexes up to now.
    props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(HConstants.LATEST_TIMESTAMP));
    Set<String> physicalTables = new HashSet<>();
    SQLException sqlEx = null;
    PhoenixConnection globalConnection = null;
    PhoenixConnection toReturn = null;
    try {
        globalConnection = new PhoenixConnection(connParam, connParam.getQueryServices(), props);
        String tenantId = null;
        try (HBaseAdmin admin = globalConnection.getQueryServices().getAdmin()) {
            String fetchViewIndexes = "SELECT " + TENANT_ID + ", " + TABLE_SCHEM + ", " + TABLE_NAME + ", " + DATA_TABLE_NAME + " FROM " + SYSTEM_CATALOG_NAME + " WHERE " + VIEW_INDEX_ID + " IS NOT NULL";
            String disableIndexDDL = "ALTER INDEX %s ON %s DISABLE";
            try (ResultSet rs = globalConnection.createStatement().executeQuery(fetchViewIndexes)) {
                while (rs.next()) {
                    tenantId = rs.getString(1);
                    String indexSchema = rs.getString(2);
                    String indexName = rs.getString(3);
                    String viewName = rs.getString(4);
                    String fullIndexName = SchemaUtil.getTableName(indexSchema, indexName);
                    String fullViewName = SchemaUtil.getTableName(indexSchema, viewName);
                    PTable viewPTable = null;
                    // Users would need to rebuild the view indexes. 
                    if (tenantId != null && !tenantId.isEmpty()) {
                        Properties newProps = PropertiesUtil.deepCopy(globalConnection.getClientInfo());
                        newProps.setProperty(PhoenixRuntime.TENANT_ID_ATTRIB, tenantId);
                        PTable indexPTable = null;
                        try (PhoenixConnection tenantConnection = new PhoenixConnection(globalConnection, globalConnection.getQueryServices(), newProps)) {
                            viewPTable = PhoenixRuntime.getTable(tenantConnection, fullViewName);
                            tenantConnection.createStatement().execute(String.format(disableIndexDDL, indexName, fullViewName));
                            indexPTable = PhoenixRuntime.getTable(tenantConnection, fullIndexName);
                        }
                        int offset = indexPTable.getBucketNum() != null ? 1 : 0;
                        // positions are stored 1 based
                        int existingTenantIdPosition = ++offset;
                        int existingViewIdxIdPosition = ++offset;
                        int newTenantIdPosition = existingViewIdxIdPosition;
                        int newViewIdxPosition = existingTenantIdPosition;
                        String tenantIdColumn = indexPTable.getColumns().get(existingTenantIdPosition - 1).getName().getString();
                        int index = 0;
                        String updatePosition = "UPSERT INTO " + SYSTEM_CATALOG_NAME + " ( " + TENANT_ID + "," + TABLE_SCHEM + "," + TABLE_NAME + "," + COLUMN_NAME + "," + COLUMN_FAMILY + "," + ORDINAL_POSITION + ") SELECT " + TENANT_ID + "," + TABLE_SCHEM + "," + TABLE_NAME + "," + COLUMN_NAME + "," + COLUMN_FAMILY + "," + "?" + " FROM " + SYSTEM_CATALOG_NAME + " WHERE " + TENANT_ID + " = ? " + " AND " + TABLE_NAME + " = ? " + " AND " + (indexSchema == null ? TABLE_SCHEM + " IS NULL" : TABLE_SCHEM + " = ? ") + " AND " + COLUMN_NAME + " = ? ";
                        // update view index position
                        try (PreparedStatement s = globalConnection.prepareStatement(updatePosition)) {
                            index = 0;
                            s.setInt(++index, newViewIdxPosition);
                            s.setString(++index, tenantId);
                            s.setString(++index, indexName);
                            if (indexSchema != null) {
                                s.setString(++index, indexSchema);
                            }
                            s.setString(++index, MetaDataUtil.getViewIndexIdColumnName());
                            s.executeUpdate();
                        }
                        // update tenant id position
                        try (PreparedStatement s = globalConnection.prepareStatement(updatePosition)) {
                            index = 0;
                            s.setInt(++index, newTenantIdPosition);
                            s.setString(++index, tenantId);
                            s.setString(++index, indexName);
                            if (indexSchema != null) {
                                s.setString(++index, indexSchema);
                            }
                            s.setString(++index, tenantIdColumn);
                            s.executeUpdate();
                        }
                        globalConnection.commit();
                    } else {
                        viewPTable = PhoenixRuntime.getTable(globalConnection, fullViewName);
                        globalConnection.createStatement().execute(String.format(disableIndexDDL, indexName, fullViewName));
                    }
                    String indexPhysicalTableName = MetaDataUtil.getViewIndexTableName(viewPTable.getPhysicalName().getString());
                    if (physicalTables.add(indexPhysicalTableName)) {
                        final TableName tableName = TableName.valueOf(indexPhysicalTableName);
                        admin.disableTable(tableName);
                        admin.truncateTable(tableName, false);
                    }
                }
            }
        }
        if (originalScn != null) {
            props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(originalScn));
        }
        toReturn = new PhoenixConnection(globalConnection, globalConnection.getQueryServices(), props);
    } catch (SQLException e) {
        sqlEx = e;
    } finally {
        sqlEx = closeConnection(connParam, sqlEx);
        sqlEx = closeConnection(globalConnection, sqlEx);
        if (sqlEx != null) {
            throw sqlEx;
        }
    }
    return toReturn;
}

Example 35 with PTable

use of org.apache.phoenix.schema.PTable in project phoenix by apache.

the class TransactionUtil method getResolvedTimestamp.

public static long getResolvedTimestamp(PhoenixConnection connection, MetaDataMutationResult result) {
    PTable table = result.getTable();
    MutationState mutationState = connection.getMutationState();
    boolean txInProgress = table != null && table.isTransactional() && mutationState.isTransactionStarted();
    return txInProgress ? convertToMilliseconds(mutationState.getInitialWritePointer()) : result.getMutationTime();
}