Examples with SplitInfos org.apache.hadoop.hive.metastore.Metastore.SplitInfos used on opensource projects

Example 1 with SplitInfos

use of org.apache.hadoop.hive.metastore.Metastore.SplitInfos in project hive by apache.

the class OrcFileFormatProxy method applySargToMetadata.

@Override
public SplitInfos applySargToMetadata(SearchArgument sarg, ByteBuffer fileMetadata) throws IOException {
    // TODO: ideally we should store shortened representation of only the necessary fields
    //       in HBase; it will probably require custom SARG application code.
    OrcTail orcTail = ReaderImpl.extractFileTail(fileMetadata);
    OrcProto.Footer footer = orcTail.getFooter();
    int stripeCount = footer.getStripesCount();
    boolean[] result = OrcInputFormat.pickStripesViaTranslatedSarg(sarg, orcTail.getWriterVersion(), footer.getTypesList(), orcTail.getStripeStatistics(), stripeCount);
    // For ORC case, send the boundaries of the stripes so we don't have to send the footer.
    SplitInfos.Builder sb = SplitInfos.newBuilder();
    List<StripeInformation> stripes = orcTail.getStripes();
    boolean isEliminated = true;
    for (int i = 0; i < result.length; ++i) {
        if (result != null && !result[i])
            continue;
        isEliminated = false;
        StripeInformation si = stripes.get(i);
        if (LOG.isDebugEnabled()) {
            LOG.debug("PPD is adding a split " + i + ": " + si.getOffset() + ", " + si.getLength());
        }
        sb.addInfos(SplitInfo.newBuilder().setIndex(i).setOffset(si.getOffset()).setLength(si.getLength()));
    }
    return isEliminated ? null : sb.build();
}

Example 2 with SplitInfos

use of org.apache.hadoop.hive.metastore.Metastore.SplitInfos in project hive by apache.

the class OrcFileMetadataHandler method getFileMetadataByExpr.

@Override
public void getFileMetadataByExpr(List<Long> fileIds, byte[] expr, ByteBuffer[] metadatas, ByteBuffer[] results, boolean[] eliminated) throws IOException {
    SearchArgument sarg = getExpressionProxy().createSarg(expr);
    // For now, don't push anything into HBase, nor store anything special in HBase
    if (metadatas == null) {
        // null means don't return metadata; we'd need the array anyway for now.
        metadatas = new ByteBuffer[results.length];
    }
    getStore().getFileMetadata(fileIds, metadatas);
    for (int i = 0; i < metadatas.length; ++i) {
        eliminated[i] = false;
        results[i] = null;
        if (metadatas[i] == null)
            continue;
        // Duplicate to avoid modification.
        ByteBuffer metadata = metadatas[i].duplicate();
        SplitInfos result = null;
        try {
            result = getFileFormatProxy().applySargToMetadata(sarg, metadata);
        } catch (IOException ex) {
            LOG.error("Failed to apply SARG to metadata", ex);
            metadatas[i] = null;
            continue;
        }
        eliminated[i] = (result == null);
        if (!eliminated[i]) {
            results[i] = ByteBuffer.wrap(result.toByteArray());
        }
    }
}