Examples with ColumnInfo org.apache.phoenix.util.ColumnInfo used on opensource projects

Example 11 with ColumnInfo

use of org.apache.phoenix.util.ColumnInfo in project phoenix by apache.

the class BaseEventSerializer method initialize.

@Override
public void initialize() throws SQLException {
    final Properties props = new Properties();
    props.setProperty(UPSERT_BATCH_SIZE_ATTRIB, String.valueOf(this.batchSize));
    ResultSet rs = null;
    try {
        this.connection = DriverManager.getConnection(this.jdbcUrl, props);
        this.connection.setAutoCommit(false);
        if (this.createTableDdl != null) {
            SchemaHandler.createTable(connection, createTableDdl);
        }
        final Map<String, Integer> qualifiedColumnMap = Maps.newLinkedHashMap();
        final Map<String, Integer> unqualifiedColumnMap = Maps.newLinkedHashMap();
        final String schemaName = SchemaUtil.getSchemaNameFromFullName(fullTableName);
        final String tableName = SchemaUtil.getTableNameFromFullName(fullTableName);
        String rowkey = null;
        String cq = null;
        String cf = null;
        Integer dt = null;
        rs = connection.getMetaData().getColumns("", StringUtil.escapeLike(SchemaUtil.normalizeIdentifier(schemaName)), StringUtil.escapeLike(SchemaUtil.normalizeIdentifier(tableName)), null);
        while (rs.next()) {
            cf = rs.getString(QueryUtil.COLUMN_FAMILY_POSITION);
            cq = rs.getString(QueryUtil.COLUMN_NAME_POSITION);
            // TODO: Fix this .. change `DATA_TYPE_POSITION` value 5 to 26
            // dt = rs.getInt(QueryUtil.DATA_TYPE_POSITION);
            dt = rs.getInt(26);
            if (Strings.isNullOrEmpty(cf)) {
                // this is required only when row key is auto generated
                rowkey = cq;
            } else {
                qualifiedColumnMap.put(SchemaUtil.getColumnDisplayName(cf, cq), dt);
            }
            unqualifiedColumnMap.put(SchemaUtil.getColumnDisplayName(null, cq), dt);
        }
        //can happen when table not found in Hbase.
        if (unqualifiedColumnMap.isEmpty()) {
            throw new SQLExceptionInfo.Builder(SQLExceptionCode.TABLE_UNDEFINED).setTableName(tableName).build().buildException();
        }
        int colSize = colNames.size();
        int headersSize = headers.size();
        int totalSize = colSize + headersSize + (autoGenerateKey ? 1 : 0);
        columnMetadata = new ColumnInfo[totalSize];
        int position = 0;
        position = this.addToColumnMetadataInfo(colNames, qualifiedColumnMap, unqualifiedColumnMap, position);
        position = this.addToColumnMetadataInfo(headers, qualifiedColumnMap, unqualifiedColumnMap, position);
        if (autoGenerateKey) {
            Integer sqlType = unqualifiedColumnMap.get(rowkey);
            if (sqlType == null) {
                throw new SQLExceptionInfo.Builder(SQLExceptionCode.PRIMARY_KEY_MISSING).setColumnName(rowkey).setTableName(fullTableName).build().buildException();
            }
            columnMetadata[position] = new ColumnInfo(rowkey, sqlType);
            position++;
        }
        this.upsertStatement = QueryUtil.constructUpsertStatement(fullTableName, Arrays.asList(columnMetadata));
        logger.info(" the upsert statement is {} ", this.upsertStatement);
    } catch (SQLException e) {
        logger.error("error {} occurred during initializing connection ", e.getMessage());
        throw e;
    } finally {
        if (rs != null) {
            rs.close();
        }
    }
    doInitialize();
}

Example 12 with ColumnInfo

use of org.apache.phoenix.util.ColumnInfo in project phoenix by apache.

the class BaseEventSerializer method addToColumnMetadataInfo.

private int addToColumnMetadataInfo(final List<String> columns, final Map<String, Integer> qualifiedColumnsInfoMap, Map<String, Integer> unqualifiedColumnsInfoMap, int position) throws SQLException {
    Preconditions.checkNotNull(columns);
    Preconditions.checkNotNull(qualifiedColumnsInfoMap);
    Preconditions.checkNotNull(unqualifiedColumnsInfoMap);
    for (int i = 0; i < columns.size(); i++) {
        String columnName = SchemaUtil.normalizeIdentifier(columns.get(i).trim());
        Integer sqlType = unqualifiedColumnsInfoMap.get(columnName);
        if (sqlType == null) {
            sqlType = qualifiedColumnsInfoMap.get(columnName);
            if (sqlType == null) {
                throw new SQLExceptionInfo.Builder(SQLExceptionCode.COLUMN_NOT_FOUND).setColumnName(columnName).setTableName(this.fullTableName).build().buildException();
            }
        }
        columnMetadata[position] = new ColumnInfo(columnName, sqlType);
        position++;
    }
    return position;
}

Example 13 with ColumnInfo

use of org.apache.phoenix.util.ColumnInfo in project phoenix by apache.

the class SqlQueryToColumnInfoFunction method apply.

@Override
public List<ColumnInfo> apply(String sqlQuery) {
    Preconditions.checkNotNull(sqlQuery);
    Connection connection = null;
    List<ColumnInfo> columnInfos = null;
    try {
        connection = ConnectionUtil.getInputConnection(this.configuration);
        final Statement statement = connection.createStatement();
        final PhoenixStatement pstmt = statement.unwrap(PhoenixStatement.class);
        final QueryPlan queryPlan = pstmt.compileQuery(sqlQuery);
        final List<? extends ColumnProjector> projectedColumns = queryPlan.getProjector().getColumnProjectors();
        columnInfos = Lists.newArrayListWithCapacity(projectedColumns.size());
        columnInfos = Lists.transform(projectedColumns, new Function<ColumnProjector, ColumnInfo>() {

            @Override
            public ColumnInfo apply(final ColumnProjector columnProjector) {
                return new ColumnInfo(columnProjector.getName(), columnProjector.getExpression().getDataType().getSqlType());
            }
        });
    } catch (SQLException e) {
        LOG.error(String.format(" Error [%s] parsing SELECT query [%s] ", e.getMessage(), sqlQuery));
        throw new RuntimeException(e);
    } finally {
        if (connection != null) {
            try {
                connection.close();
            } catch (SQLException sqle) {
                LOG.error("Error closing connection!!");
                throw new RuntimeException(sqle);
            }
        }
    }
    return columnInfos;
}

Example 14 with ColumnInfo

use of org.apache.phoenix.util.ColumnInfo in project phoenix by apache.

the class AbstractBulkLoadTool method loadData.

private int loadData(Configuration conf, CommandLine cmdLine) throws Exception {
    String tableName = cmdLine.getOptionValue(TABLE_NAME_OPT.getOpt());
    String schemaName = cmdLine.getOptionValue(SCHEMA_NAME_OPT.getOpt());
    String indexTableName = cmdLine.getOptionValue(INDEX_TABLE_NAME_OPT.getOpt());
    String qualifiedTableName = SchemaUtil.getQualifiedTableName(schemaName, tableName);
    String qualifiedIndexTableName = null;
    if (indexTableName != null) {
        qualifiedIndexTableName = SchemaUtil.getQualifiedTableName(schemaName, indexTableName);
    }
    if (cmdLine.hasOption(ZK_QUORUM_OPT.getOpt())) {
        // ZK_QUORUM_OPT is optional, but if it's there, use it for both the conn and the job.
        String zkQuorum = cmdLine.getOptionValue(ZK_QUORUM_OPT.getOpt());
        PhoenixDriver.ConnectionInfo info = PhoenixDriver.ConnectionInfo.create(zkQuorum);
        LOG.info("Configuring HBase connection to {}", info);
        for (Map.Entry<String, String> entry : info.asProps()) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("Setting {} = {}", entry.getKey(), entry.getValue());
            }
            conf.set(entry.getKey(), entry.getValue());
        }
    }
    final Connection conn = QueryUtil.getConnection(conf);
    if (LOG.isDebugEnabled()) {
        LOG.debug("Reading columns from {} :: {}", ((PhoenixConnection) conn).getURL(), qualifiedTableName);
    }
    List<ColumnInfo> importColumns = buildImportColumns(conn, cmdLine, qualifiedTableName);
    Preconditions.checkNotNull(importColumns);
    Preconditions.checkArgument(!importColumns.isEmpty(), "Column info list is empty");
    FormatToBytesWritableMapper.configureColumnInfoList(conf, importColumns);
    boolean ignoreInvalidRows = cmdLine.hasOption(IGNORE_ERRORS_OPT.getOpt());
    conf.setBoolean(FormatToBytesWritableMapper.IGNORE_INVALID_ROW_CONFKEY, ignoreInvalidRows);
    conf.set(FormatToBytesWritableMapper.TABLE_NAME_CONFKEY, qualifiedTableName);
    // give subclasses their hook
    configureOptions(cmdLine, importColumns, conf);
    try {
        validateTable(conn, schemaName, tableName);
    } finally {
        conn.close();
    }
    final String inputPaths = cmdLine.getOptionValue(INPUT_PATH_OPT.getOpt());
    final Path outputPath;
    if (cmdLine.hasOption(OUTPUT_PATH_OPT.getOpt())) {
        outputPath = new Path(cmdLine.getOptionValue(OUTPUT_PATH_OPT.getOpt()));
    } else {
        outputPath = new Path("/tmp/" + UUID.randomUUID());
    }
    List<TargetTableRef> tablesToBeLoaded = new ArrayList<TargetTableRef>();
    PTable table = PhoenixRuntime.getTable(conn, qualifiedTableName);
    tablesToBeLoaded.add(new TargetTableRef(qualifiedTableName, table.getPhysicalName().getString()));
    boolean hasLocalIndexes = false;
    for (PTable index : table.getIndexes()) {
        if (index.getIndexType() == IndexType.LOCAL) {
            hasLocalIndexes = qualifiedIndexTableName == null ? true : index.getTableName().getString().equals(qualifiedIndexTableName);
            if (hasLocalIndexes)
                break;
        }
    }
    // using conn after it's been closed... o.O
    tablesToBeLoaded.addAll(getIndexTables(conn, schemaName, qualifiedTableName));
    // When loading a single index table, check index table name is correct
    if (qualifiedIndexTableName != null) {
        TargetTableRef targetIndexRef = null;
        for (TargetTableRef tmpTable : tablesToBeLoaded) {
            if (tmpTable.getLogicalName().compareToIgnoreCase(qualifiedIndexTableName) == 0) {
                targetIndexRef = tmpTable;
                break;
            }
        }
        if (targetIndexRef == null) {
            throw new IllegalStateException("Bulk Loader error: index table " + qualifiedIndexTableName + " doesn't exist");
        }
        tablesToBeLoaded.clear();
        tablesToBeLoaded.add(targetIndexRef);
    }
    return submitJob(conf, tableName, inputPaths, outputPath, tablesToBeLoaded, hasLocalIndexes);
}

Example 15 with ColumnInfo

use of org.apache.phoenix.util.ColumnInfo in project phoenix by apache.

the class PhoenixPigSchemaUtil method getResourceSchema.

public static ResourceSchema getResourceSchema(final Configuration configuration, Dependencies dependencies) throws IOException {
    final ResourceSchema schema = new ResourceSchema();
    try {
        List<ColumnInfo> columns = null;
        final SchemaType schemaType = PhoenixConfigurationUtil.getSchemaType(configuration);
        if (SchemaType.QUERY.equals(schemaType)) {
            final String sqlQuery = PhoenixConfigurationUtil.getSelectStatement(configuration);
            Preconditions.checkNotNull(sqlQuery, "No Sql Query exists within the configuration");
            final SqlQueryToColumnInfoFunction function = new SqlQueryToColumnInfoFunction(configuration);
            columns = function.apply(sqlQuery);
        } else {
            columns = dependencies.getSelectColumnMetadataList(configuration);
        }
        ResourceFieldSchema[] fields = new ResourceFieldSchema[columns.size()];
        int i = 0;
        for (ColumnInfo cinfo : columns) {
            int sqlType = cinfo.getSqlType();
            PDataType phoenixDataType = PDataType.fromTypeId(sqlType);
            byte pigType = TypeUtil.getPigDataTypeForPhoenixType(phoenixDataType);
            ResourceFieldSchema field = new ResourceFieldSchema();
            field.setType(pigType).setName(cinfo.getDisplayName());
            fields[i++] = field;
        }
        schema.setFields(fields);
    } catch (SQLException sqle) {
        LOG.error(String.format("Error: SQLException [%s] ", sqle.getMessage()));
        throw new IOException(sqle);
    }
    return schema;
}