Examples with Subfield com.facebook.presto.common.Subfield used on opensource projects

Example 11 with Subfield

use of com.facebook.presto.common.Subfield in project presto by prestodb.

the class ListFilter method extractFilters.

private static long[] extractFilters(Map<Subfield, TupleDomainFilter> filters, int level, Predicate<Subfield> predicate) {
    int maxSubscript = -1;
    int[] subscripts = new int[filters.size()];
    int index = 0;
    for (Subfield subfield : filters.keySet()) {
        if (predicate.test(subfield)) {
            subscripts[index] = toSubscript(subfield, level);
            maxSubscript = Math.max(maxSubscript, subscripts[index]);
        } else {
            subscripts[index] = -1;
        }
        index++;
    }
    if (maxSubscript == -1) {
        return null;
    }
    long[] filterCodes = new long[maxSubscript + 1];
    for (int i = 0; i < subscripts.length; i++) {
        if (subscripts[i] != -1) {
            filterCodes[subscripts[i]] |= 1 << i;
        }
    }
    return filterCodes;
}

Example 12 with Subfield

use of com.facebook.presto.common.Subfield in project presto by prestodb.

the class AbstractOrcRecordReader method includeOrcColumnsRecursive.

private static void includeOrcColumnsRecursive(List<OrcType> types, Set<Integer> result, int typeId, List<Subfield> requiredSubfields) {
    result.add(typeId);
    OrcType type = types.get(typeId);
    Optional<Map<String, List<Subfield>>> requiredFields = Optional.empty();
    if (type.getOrcTypeKind() == STRUCT) {
        requiredFields = getRequiredFields(requiredSubfields);
    }
    int children = type.getFieldCount();
    for (int i = 0; i < children; ++i) {
        List<Subfield> subfields = ImmutableList.of();
        if (requiredFields.isPresent()) {
            String fieldName = type.getFieldNames().get(i).toLowerCase(Locale.ENGLISH);
            if (!requiredFields.get().containsKey(fieldName)) {
                continue;
            }
            subfields = requiredFields.get().get(fieldName);
        }
        includeOrcColumnsRecursive(types, result, type.getFieldTypeIndex(i), subfields);
    }
}

Example 13 with Subfield

use of com.facebook.presto.common.Subfield in project presto by prestodb.

the class OrcSelectiveRecordReader method createStreamReaders.

private static SelectiveStreamReader[] createStreamReaders(OrcDataSource orcDataSource, List<OrcType> types, DateTimeZone hiveStorageTimeZone, OrcRecordReaderOptions options, boolean legacyMapSubscript, Map<Integer, Type> includedColumns, List<Integer> outputColumns, Map<Integer, Map<Subfield, TupleDomainFilter>> filters, List<FilterFunction> filterFunctions, Map<Integer, Integer> filterFunctionInputMapping, Map<Integer, List<Subfield>> requiredSubfields, OrcAggregatedMemoryContext systemMemoryContext) {
    List<StreamDescriptor> streamDescriptors = createStreamDescriptor("", "", 0, types, orcDataSource).getNestedStreams();
    requireNonNull(filterFunctions, "filterFunctions is null");
    requireNonNull(filterFunctionInputMapping, "filterFunctionInputMapping is null");
    Set<Integer> filterFunctionInputColumns = filterFunctions.stream().flatMapToInt(function -> Arrays.stream(function.getInputChannels())).boxed().map(filterFunctionInputMapping::get).collect(toImmutableSet());
    OrcType rowType = types.get(0);
    SelectiveStreamReader[] streamReaders = new SelectiveStreamReader[rowType.getFieldCount()];
    for (int columnId = 0; columnId < rowType.getFieldCount(); columnId++) {
        if (includedColumns.containsKey(columnId)) {
            StreamDescriptor streamDescriptor = streamDescriptors.get(columnId);
            boolean outputRequired = outputColumns.contains(columnId) || filterFunctionInputColumns.contains(columnId);
            streamReaders[columnId] = createStreamReader(streamDescriptor, Optional.ofNullable(filters.get(columnId)).orElse(ImmutableMap.of()), outputRequired ? Optional.of(includedColumns.get(columnId)) : Optional.empty(), Optional.ofNullable(requiredSubfields.get(columnId)).orElse(ImmutableList.of()), hiveStorageTimeZone, options, legacyMapSubscript, systemMemoryContext);
        }
    }
    return streamReaders;
}

Example 14 with Subfield

use of com.facebook.presto.common.Subfield in project presto by prestodb.

the class OrcSelectiveRecordReader method scoreFilter.

private static int scoreFilter(Map<Subfield, TupleDomainFilter> filters) {
    checkArgument(!filters.isEmpty());
    if (filters.size() > 1) {
        // Complex type column. Complex types are expensive!
        return 1000;
    }
    Map.Entry<Subfield, TupleDomainFilter> filterEntry = Iterables.getOnlyElement(filters.entrySet());
    if (!filterEntry.getKey().getPath().isEmpty()) {
        // Complex type column. Complex types are expensive!
        return 1000;
    }
    TupleDomainFilter filter = filterEntry.getValue();
    if (filter instanceof BigintRange) {
        if (((BigintRange) filter).isSingleValue()) {
            // Integer equality. Generally cheap.
            return 10;
        }
        return 50;
    }
    if (filter instanceof BigintValuesUsingHashTable || filter instanceof BigintValuesUsingBitmask || filter instanceof BigintMultiRange) {
        return 50;
    }
    return 100;
}

Example 15 with Subfield

use of com.facebook.presto.common.Subfield in project presto by prestodb.

the class TestDecryption method testReadPermittedColumnsWithoutAllKeys.

@Test
public void testReadPermittedColumnsWithoutAllKeys() throws Exception {
    Subfield subfield1 = new Subfield("c.field_0");
    Subfield subfield3 = new Subfield("c.field_2");
    Type rowType = rowType(BIGINT, BIGINT, BIGINT);
    Slice iek1 = Slices.utf8Slice("iek1");
    Slice iek2 = Slices.utf8Slice("iek2");
    Slice iek3 = Slices.utf8Slice("iek3");
    DwrfWriterEncryption dwrfWriterEncryption = new DwrfWriterEncryption(UNKNOWN, ImmutableList.of(new WriterEncryptionGroup(ImmutableList.of(2), iek1), new WriterEncryptionGroup(ImmutableList.of(3, 6), iek2), new WriterEncryptionGroup(ImmutableList.of(5), iek3)));
    List<Type> types = ImmutableList.of(rowType, BIGINT, BIGINT, BIGINT);
    List<Long> columnValues = ImmutableList.copyOf(intsBetween(0, 31_234)).stream().map(Number::longValue).collect(toList());
    List<List<?>> writtenValues = ImmutableList.of(columnValues.stream().map(OrcTester::toHiveStruct).collect(toList()), columnValues, columnValues, columnValues);
    List<List<?>> readValues = ImmutableList.of(columnValues.stream().map(value -> asList(value, null, value)).collect(toList()), columnValues, columnValues);
    testDecryptionRoundTrip(types, writtenValues, readValues, Optional.of(dwrfWriterEncryption), ImmutableMap.of(2, iek1, 5, iek3), ImmutableMap.of(0, rowType, 1, BIGINT, 3, BIGINT), ImmutableMap.of(0, ImmutableList.of(subfield1, subfield3)), ImmutableList.of(0, 1, 3));
}