Examples with Subfield com.facebook.presto.common.Subfield used on opensource projects

Example 46 with Subfield

use of com.facebook.presto.common.Subfield in project presto by prestodb.

the class TestAbstractDwrfEncryptionInformationSource method testGetReadEncryptionInformationForUnPartitionedTableWithColumnLevelEncryption.

@Test
public void testGetReadEncryptionInformationForUnPartitionedTableWithColumnLevelEncryption() {
    Table table = createTable(DWRF, Optional.of(forPerColumn(fromHiveProperty("key1:col_string,col_struct.b.b2;key2:col_bigint,col_struct.a"), "algo", "provider")), false);
    Optional<EncryptionInformation> encryptionInformation = encryptionInformationSource.getReadEncryptionInformation(SESSION, table, Optional.of(ImmutableSet.of(// hiveColumnIndex value does not matter in this test
    new HiveColumnHandle("col_bigint", HIVE_LONG, HIVE_LONG.getTypeSignature(), 0, REGULAR, Optional.empty(), Optional.empty()), new HiveColumnHandle("col_map", HIVE_LONG, HIVE_LONG.getTypeSignature(), 0, REGULAR, Optional.empty(), Optional.empty()), new HiveColumnHandle("col_struct", STRUCT_TYPE, STRUCT_TYPE.getTypeSignature(), 0, REGULAR, Optional.empty(), ImmutableList.of(new Subfield("col_struct.a"), new Subfield("col_struct.b.b2")), Optional.empty()))));
    Map<String, byte[]> expectedFieldToKeyData = ImmutableMap.of("col_bigint", "key2".getBytes(), "col_struct.a", "key2".getBytes(), "col_struct.b.b2", "key1".getBytes());
    assertTrue(encryptionInformation.isPresent());
    assertEquals(encryptionInformation.get(), EncryptionInformation.fromEncryptionMetadata(DwrfEncryptionMetadata.forPerField(expectedFieldToKeyData, ImmutableMap.of(TEST_EXTRA_METADATA, table.getTableName()), "algo", "provider")));
}

Example 47 with Subfield

use of com.facebook.presto.common.Subfield in project presto by prestodb.

the class TestCoercingFilters method testIntegerToVarchar.

@Test
public void testIntegerToVarchar() {
    TupleDomainFilter filter = BytesRange.of("10".getBytes(), false, "10".getBytes(), false, false);
    HiveCoercer coercer = new IntegerNumberToVarcharCoercer(INTEGER, VARCHAR);
    TupleDomainFilter coercingFilter = coercer.toCoercingFilter(filter, new Subfield("c"));
    assertTrue(coercingFilter.testLong(10));
    assertFalse(coercingFilter.testLong(25));
    assertFalse(coercingFilter.testNull());
}

Example 48 with Subfield

use of com.facebook.presto.common.Subfield in project presto by prestodb.

the class TestCoercingFilters method testVarcharToInteger.

@Test
public void testVarcharToInteger() {
    TupleDomainFilter filter = BigintRange.of(100, Integer.MAX_VALUE, false);
    HiveCoercer coercer = new VarcharToIntegerNumberCoercer(VARCHAR, INTEGER);
    TupleDomainFilter coercingFilter = coercer.toCoercingFilter(filter, new Subfield("c"));
    assertTrue(coercingFilter.testLength(1));
    assertTrue(coercingFilter.testLength(2));
    assertTrue(coercingFilter.testLength(3));
    assertTrue(coercingFilter.testBytes("100".getBytes(), 0, 3));
    assertTrue(coercingFilter.testBytes("145".getBytes(), 0, 3));
    assertTrue(coercingFilter.testBytes("2147483647".getBytes(), 0, 10));
    assertFalse(coercingFilter.testBytes("50".getBytes(), 0, 2));
    assertFalse(coercingFilter.testBytes("-50".getBytes(), 0, 3));
    // parsing error
    assertFalse(coercingFilter.testBytes("abc".getBytes(), 0, 3));
    // out of range
    assertFalse(coercingFilter.testBytes("2147483648".getBytes(), 0, 10));
    assertFalse(coercingFilter.testNull());
}

Example 49 with Subfield

use of com.facebook.presto.common.Subfield in project presto by prestodb.

the class SelectiveStreamReaders method createStreamReader.

public static SelectiveStreamReader createStreamReader(StreamDescriptor streamDescriptor, Map<Subfield, TupleDomainFilter> filters, Optional<Type> outputType, List<Subfield> requiredSubfields, DateTimeZone hiveStorageTimeZone, OrcRecordReaderOptions options, boolean legacyMapSubscript, OrcAggregatedMemoryContext systemMemoryContext) {
    OrcTypeKind type = streamDescriptor.getOrcTypeKind();
    switch(type) {
        case BOOLEAN:
            {
                checkArgument(requiredSubfields.isEmpty(), "Boolean stream reader doesn't support subfields");
                verifyStreamType(streamDescriptor, outputType, BooleanType.class::isInstance);
                return new BooleanSelectiveStreamReader(streamDescriptor, getOptionalOnlyFilter(type, filters), outputType.isPresent(), systemMemoryContext.newOrcLocalMemoryContext(SelectiveStreamReaders.class.getSimpleName()));
            }
        case BYTE:
            {
                checkArgument(requiredSubfields.isEmpty(), "Byte stream reader doesn't support subfields");
                verifyStreamType(streamDescriptor, outputType, TinyintType.class::isInstance);
                return new ByteSelectiveStreamReader(streamDescriptor, getOptionalOnlyFilter(type, filters), outputType.isPresent(), systemMemoryContext.newOrcLocalMemoryContext(SelectiveStreamReaders.class.getSimpleName()));
            }
        case SHORT:
        case INT:
        case LONG:
        case DATE:
            {
                checkArgument(requiredSubfields.isEmpty(), "Primitive type stream reader doesn't support subfields");
                verifyStreamType(streamDescriptor, outputType, t -> t instanceof BigintType || t instanceof IntegerType || t instanceof SmallintType || t instanceof DateType);
                return new LongSelectiveStreamReader(streamDescriptor, getOptionalOnlyFilter(type, filters), outputType, systemMemoryContext);
            }
        case FLOAT:
            {
                checkArgument(requiredSubfields.isEmpty(), "Float type stream reader doesn't support subfields");
                verifyStreamType(streamDescriptor, outputType, RealType.class::isInstance);
                return new FloatSelectiveStreamReader(streamDescriptor, getOptionalOnlyFilter(type, filters), outputType.isPresent(), systemMemoryContext.newOrcLocalMemoryContext(SelectiveStreamReaders.class.getSimpleName()));
            }
        case DOUBLE:
            checkArgument(requiredSubfields.isEmpty(), "Double stream reader doesn't support subfields");
            verifyStreamType(streamDescriptor, outputType, DoubleType.class::isInstance);
            return new DoubleSelectiveStreamReader(streamDescriptor, getOptionalOnlyFilter(type, filters), outputType.isPresent(), systemMemoryContext.newOrcLocalMemoryContext(SelectiveStreamReaders.class.getSimpleName()));
        case BINARY:
        case STRING:
        case VARCHAR:
        case CHAR:
            checkArgument(requiredSubfields.isEmpty(), "Primitive stream reader doesn't support subfields");
            verifyStreamType(streamDescriptor, outputType, t -> t instanceof VarcharType || t instanceof CharType || t instanceof VarbinaryType);
            return new SliceSelectiveStreamReader(streamDescriptor, getOptionalOnlyFilter(type, filters), outputType, systemMemoryContext);
        case TIMESTAMP:
            {
                checkArgument(requiredSubfields.isEmpty(), "Timestamp stream reader doesn't support subfields");
                verifyStreamType(streamDescriptor, outputType, TimestampType.class::isInstance);
                return new TimestampSelectiveStreamReader(streamDescriptor, getOptionalOnlyFilter(type, filters), hiveStorageTimeZone, outputType.isPresent(), systemMemoryContext.newOrcLocalMemoryContext(SelectiveStreamReaders.class.getSimpleName()), options);
            }
        case LIST:
            verifyStreamType(streamDescriptor, outputType, ArrayType.class::isInstance);
            return new ListSelectiveStreamReader(streamDescriptor, filters, requiredSubfields, null, 0, outputType, hiveStorageTimeZone, options, legacyMapSubscript, systemMemoryContext);
        case STRUCT:
            verifyStreamType(streamDescriptor, outputType, RowType.class::isInstance);
            return new StructSelectiveStreamReader(streamDescriptor, filters, requiredSubfields, outputType, hiveStorageTimeZone, options, legacyMapSubscript, systemMemoryContext);
        case MAP:
            verifyStreamType(streamDescriptor, outputType, MapType.class::isInstance);
            return new MapSelectiveStreamReader(streamDescriptor, filters, requiredSubfields, outputType, hiveStorageTimeZone, options, legacyMapSubscript, systemMemoryContext);
        case DECIMAL:
            {
                verifyStreamType(streamDescriptor, outputType, DecimalType.class::isInstance);
                if (streamDescriptor.getOrcType().getPrecision().get() <= MAX_SHORT_PRECISION) {
                    return new ShortDecimalSelectiveStreamReader(streamDescriptor, getOptionalOnlyFilter(type, filters), outputType, systemMemoryContext.newOrcLocalMemoryContext(SelectiveStreamReaders.class.getSimpleName()));
                } else {
                    return new LongDecimalSelectiveStreamReader(streamDescriptor, getOptionalOnlyFilter(type, filters), outputType, systemMemoryContext.newOrcLocalMemoryContext(SelectiveStreamReaders.class.getSimpleName()));
                }
            }
        case UNION:
        default:
            throw new IllegalArgumentException("Unsupported type: " + type);
    }
}

Example 50 with Subfield

use of com.facebook.presto.common.Subfield in project presto by prestodb.

the class StructSelectiveStreamReader method getRequiredFields.

private static Optional<Map<String, List<Subfield>>> getRequiredFields(List<Subfield> requiredSubfields) {
    if (requiredSubfields.isEmpty()) {
        return Optional.empty();
    }
    Map<String, List<Subfield>> fields = new HashMap<>();
    for (Subfield subfield : requiredSubfields) {
        List<Subfield.PathElement> path = subfield.getPath();
        String name = ((Subfield.NestedField) path.get(0)).getName().toLowerCase(Locale.ENGLISH);
        fields.computeIfAbsent(name, k -> new ArrayList<>());
        if (path.size() > 1) {
            fields.get(name).add(new Subfield("c", path.subList(1, path.size())));
        }
    }
    return Optional.of(ImmutableMap.copyOf(fields));
}