Examples with TypeSignature com.facebook.presto.common.type.TypeSignature used on opensource projects

Example 1 with TypeSignature

use of com.facebook.presto.common.type.TypeSignature in project presto by prestodb.

the class MongoSession method guessTableFields.

private List<Document> guessTableFields(SchemaTableName schemaTableName) {
    String schemaName = schemaTableName.getSchemaName();
    String tableName = schemaTableName.getTableName();
    MongoDatabase db = client.getDatabase(schemaName);
    Document doc = db.getCollection(tableName).find().first();
    if (doc == null) {
        // no records at the collection
        return ImmutableList.of();
    }
    ImmutableList.Builder<Document> builder = ImmutableList.builder();
    for (String key : doc.keySet()) {
        Object value = doc.get(key);
        Optional<TypeSignature> fieldType = guessFieldType(value);
        if (fieldType.isPresent()) {
            Document metadata = new Document();
            metadata.append(FIELDS_NAME_KEY, key);
            metadata.append(FIELDS_TYPE_KEY, fieldType.get().toString());
            metadata.append(FIELDS_HIDDEN_KEY, key.equals("_id") && fieldType.get().equals(OBJECT_ID.getTypeSignature()));
            builder.add(metadata);
        } else {
            log.debug("Unable to guess field type from %s : %s", value == null ? "null" : value.getClass().getName(), value);
        }
    }
    return builder.build();
}

Example 2 with TypeSignature

use of com.facebook.presto.common.type.TypeSignature in project presto by prestodb.

the class TestShardWriter method testWriter.

@Test(dataProvider = "useOptimizedOrcWriter")
public void testWriter(boolean useOptimizedOrcWriter) throws Exception {
    FunctionAndTypeManager functionAndTypeManager = createTestFunctionAndTypeManager();
    List<Long> columnIds = ImmutableList.of(1L, 2L, 4L, 6L, 7L, 8L, 9L, 10L, 11L, 12L, 13L);
    ArrayType arrayType = new ArrayType(BIGINT);
    ArrayType arrayOfArrayType = new ArrayType(arrayType);
    Type mapType = functionAndTypeManager.getParameterizedType(StandardTypes.MAP, ImmutableList.of(TypeSignatureParameter.of(createVarcharType(10).getTypeSignature()), TypeSignatureParameter.of(BOOLEAN.getTypeSignature())));
    List<Type> columnTypes = ImmutableList.of(BIGINT, createVarcharType(10), VARBINARY, DOUBLE, BOOLEAN, arrayType, mapType, arrayOfArrayType, TIMESTAMP, TIME, DATE);
    File file = new File(directory, System.nanoTime() + ".orc");
    byte[] bytes1 = octets(0x00, 0xFE, 0xFF);
    byte[] bytes3 = octets(0x01, 0x02, 0x19, 0x80);
    long timestampValue = sqlTimestampOf(2002, 4, 6, 7, 8, 9, 0, UTC, UTC_KEY, SESSION).getMillisUtc();
    long timeValue = new SqlTime(NANOSECONDS.toMillis(new DateTime(2004, 11, 29, 0, 0, 0, 0, UTC).toLocalTime().getMillisOfDay())).getMillis();
    DateTime date = new DateTime(2001, 11, 22, 0, 0, 0, 0, UTC);
    int dateValue = new SqlDate(Days.daysBetween(new DateTime(0, ISOChronology.getInstanceUTC()), date).getDays()).getDays();
    RowPagesBuilder rowPagesBuilder = RowPagesBuilder.rowPagesBuilder(columnTypes).row(123L, "hello", wrappedBuffer(bytes1), 123.456, true, arrayBlockOf(BIGINT, 1, 2), mapBlockOf(createVarcharType(5), BOOLEAN, "k1", true), arrayBlockOf(arrayType, arrayBlockOf(BIGINT, 5)), timestampValue, timeValue, dateValue).row(null, "world", null, Double.POSITIVE_INFINITY, null, arrayBlockOf(BIGINT, 3, null), mapBlockOf(createVarcharType(5), BOOLEAN, "k2", null), arrayBlockOf(arrayType, null, arrayBlockOf(BIGINT, 6, 7)), timestampValue, timeValue, dateValue).row(456L, "bye \u2603", wrappedBuffer(bytes3), Double.NaN, false, arrayBlockOf(BIGINT), mapBlockOf(createVarcharType(5), BOOLEAN, "k3", false), arrayBlockOf(arrayType, arrayBlockOf(BIGINT)), timestampValue, timeValue, dateValue);
    try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(new EmptyClassLoader());
        FileWriter writer = createFileWriter(columnIds, columnTypes, file)) {
        writer.appendPages(rowPagesBuilder.build());
    }
    try (OrcDataSource dataSource = fileOrcDataSource(file)) {
        OrcBatchRecordReader reader = createReader(dataSource, columnIds, columnTypes);
        assertEquals(reader.getReaderRowCount(), 3);
        assertEquals(reader.getReaderPosition(), 0);
        assertEquals(reader.getFileRowCount(), reader.getReaderRowCount());
        assertEquals(reader.getFilePosition(), reader.getFilePosition());
        assertEquals(reader.nextBatch(), 3);
        assertEquals(reader.getReaderPosition(), 0);
        assertEquals(reader.getFilePosition(), reader.getFilePosition());
        Block column0 = reader.readBlock(0);
        assertEquals(column0.isNull(0), false);
        assertEquals(column0.isNull(1), true);
        assertEquals(column0.isNull(2), false);
        assertEquals(BIGINT.getLong(column0, 0), 123L);
        assertEquals(BIGINT.getLong(column0, 2), 456L);
        Block column1 = reader.readBlock(1);
        assertEquals(createVarcharType(10).getSlice(column1, 0), utf8Slice("hello"));
        assertEquals(createVarcharType(10).getSlice(column1, 1), utf8Slice("world"));
        assertEquals(createVarcharType(10).getSlice(column1, 2), utf8Slice("bye \u2603"));
        Block column2 = reader.readBlock(2);
        assertEquals(VARBINARY.getSlice(column2, 0), wrappedBuffer(bytes1));
        assertEquals(column2.isNull(1), true);
        assertEquals(VARBINARY.getSlice(column2, 2), wrappedBuffer(bytes3));
        Block column3 = reader.readBlock(3);
        assertEquals(column3.isNull(0), false);
        assertEquals(column3.isNull(1), false);
        assertEquals(column3.isNull(2), false);
        assertEquals(DOUBLE.getDouble(column3, 0), 123.456);
        assertEquals(DOUBLE.getDouble(column3, 1), Double.POSITIVE_INFINITY);
        assertEquals(DOUBLE.getDouble(column3, 2), Double.NaN);
        Block column4 = reader.readBlock(4);
        assertEquals(column4.isNull(0), false);
        assertEquals(column4.isNull(1), true);
        assertEquals(column4.isNull(2), false);
        assertEquals(BOOLEAN.getBoolean(column4, 0), true);
        assertEquals(BOOLEAN.getBoolean(column4, 2), false);
        Block column5 = reader.readBlock(5);
        assertEquals(column5.getPositionCount(), 3);
        assertTrue(arrayBlocksEqual(BIGINT, arrayType.getObject(column5, 0), arrayBlockOf(BIGINT, 1, 2)));
        assertTrue(arrayBlocksEqual(BIGINT, arrayType.getObject(column5, 1), arrayBlockOf(BIGINT, 3, null)));
        assertTrue(arrayBlocksEqual(BIGINT, arrayType.getObject(column5, 2), arrayBlockOf(BIGINT)));
        Block column6 = reader.readBlock(6);
        assertEquals(column6.getPositionCount(), 3);
        assertTrue(mapBlocksEqual(createVarcharType(5), BOOLEAN, arrayType.getObject(column6, 0), mapBlockOf(createVarcharType(5), BOOLEAN, "k1", true)));
        Block object = arrayType.getObject(column6, 1);
        Block k2 = mapBlockOf(createVarcharType(5), BOOLEAN, "k2", null);
        assertTrue(mapBlocksEqual(createVarcharType(5), BOOLEAN, object, k2));
        assertTrue(mapBlocksEqual(createVarcharType(5), BOOLEAN, arrayType.getObject(column6, 2), mapBlockOf(createVarcharType(5), BOOLEAN, "k3", false)));
        Block column7 = reader.readBlock(7);
        assertEquals(column7.getPositionCount(), 3);
        assertTrue(arrayBlocksEqual(arrayType, arrayOfArrayType.getObject(column7, 0), arrayBlockOf(arrayType, arrayBlockOf(BIGINT, 5))));
        assertTrue(arrayBlocksEqual(arrayType, arrayOfArrayType.getObject(column7, 1), arrayBlockOf(arrayType, null, arrayBlockOf(BIGINT, 6, 7))));
        assertTrue(arrayBlocksEqual(arrayType, arrayOfArrayType.getObject(column7, 2), arrayBlockOf(arrayType, arrayBlockOf(BIGINT))));
        Block column8 = reader.readBlock(8);
        assertEquals(TIMESTAMP.getLong(column8, 0), timestampValue);
        assertEquals(TIMESTAMP.getLong(column8, 1), timestampValue);
        assertEquals(TIMESTAMP.getLong(column8, 2), timestampValue);
        Block column9 = reader.readBlock(9);
        assertEquals(TIME.getLong(column9, 0), timeValue);
        assertEquals(TIME.getLong(column9, 1), timeValue);
        assertEquals(TIME.getLong(column9, 2), timeValue);
        Block column10 = reader.readBlock(10);
        assertEquals(DATE.getLong(column10, 0), dateValue);
        assertEquals(DATE.getLong(column10, 1), dateValue);
        assertEquals(DATE.getLong(column10, 2), dateValue);
        assertEquals(reader.nextBatch(), -1);
        assertEquals(reader.getReaderPosition(), 3);
        assertEquals(reader.getFilePosition(), reader.getFilePosition());
        OrcFileMetadata orcFileMetadata = METADATA_CODEC.fromJson(reader.getUserMetadata().get(OrcFileMetadata.KEY).getBytes());
        assertEquals(orcFileMetadata, new OrcFileMetadata(ImmutableMap.<Long, TypeSignature>builder().put(1L, BIGINT.getTypeSignature()).put(2L, createVarcharType(10).getTypeSignature()).put(4L, VARBINARY.getTypeSignature()).put(6L, DOUBLE.getTypeSignature()).put(7L, BOOLEAN.getTypeSignature()).put(8L, arrayType.getTypeSignature()).put(9L, mapType.getTypeSignature()).put(10L, arrayOfArrayType.getTypeSignature()).put(11L, TIMESTAMP.getTypeSignature()).put(12L, TIME.getTypeSignature()).put(13L, DATE.getTypeSignature()).build()));
    }
    File crcFile = new File(file.getParentFile(), "." + file.getName() + ".crc");
    assertFalse(crcFile.exists());
    // Test unsupported types
    for (Type type : ImmutableList.of(TIMESTAMP_WITH_TIME_ZONE, RowType.anonymous(ImmutableList.of(BIGINT, DOUBLE)))) {
        try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(new EmptyClassLoader())) {
            createFileWriter(ImmutableList.of(1L), ImmutableList.of(type), file);
            fail();
        } catch (PrestoException e) {
            assertTrue(e.getMessage().toLowerCase(ENGLISH).contains("type"));
        }
    }
}

Example 3 with TypeSignature

use of com.facebook.presto.common.type.TypeSignature in project presto by prestodb.

the class BuiltInTypeAndFunctionNamespaceManager method doGetSpecializedFunctionKey.

private SpecializedFunctionKey doGetSpecializedFunctionKey(Signature signature) {
    Iterable<SqlFunction> candidates = getFunctions(null, signature.getName());
    // search for exact match
    Type returnType = functionAndTypeManager.getType(signature.getReturnType());
    List<TypeSignatureProvider> argumentTypeSignatureProviders = fromTypeSignatures(signature.getArgumentTypes());
    for (SqlFunction candidate : candidates) {
        Optional<BoundVariables> boundVariables = new SignatureBinder(functionAndTypeManager, candidate.getSignature(), false).bindVariables(argumentTypeSignatureProviders, returnType);
        if (boundVariables.isPresent()) {
            return new SpecializedFunctionKey(candidate, boundVariables.get(), argumentTypeSignatureProviders.size());
        }
    }
    // TODO: hack because there could be "type only" coercions (which aren't necessarily included as implicit casts),
    // so do a second pass allowing "type only" coercions
    List<Type> argumentTypes = resolveTypes(signature.getArgumentTypes(), functionAndTypeManager);
    for (SqlFunction candidate : candidates) {
        SignatureBinder binder = new SignatureBinder(functionAndTypeManager, candidate.getSignature(), true);
        Optional<BoundVariables> boundVariables = binder.bindVariables(argumentTypeSignatureProviders, returnType);
        if (!boundVariables.isPresent()) {
            continue;
        }
        Signature boundSignature = applyBoundVariables(candidate.getSignature(), boundVariables.get(), argumentTypes.size());
        if (!functionAndTypeManager.isTypeOnlyCoercion(functionAndTypeManager.getType(boundSignature.getReturnType()), returnType)) {
            continue;
        }
        boolean nonTypeOnlyCoercion = false;
        for (int i = 0; i < argumentTypes.size(); i++) {
            Type expectedType = functionAndTypeManager.getType(boundSignature.getArgumentTypes().get(i));
            if (!functionAndTypeManager.isTypeOnlyCoercion(argumentTypes.get(i), expectedType)) {
                nonTypeOnlyCoercion = true;
                break;
            }
        }
        if (nonTypeOnlyCoercion) {
            continue;
        }
        return new SpecializedFunctionKey(candidate, boundVariables.get(), argumentTypes.size());
    }
    // TODO: this is a hack and should be removed
    if (signature.getNameSuffix().startsWith(MAGIC_LITERAL_FUNCTION_PREFIX)) {
        List<TypeSignature> parameterTypes = signature.getArgumentTypes();
        // extract type from function name
        String typeName = signature.getNameSuffix().substring(MAGIC_LITERAL_FUNCTION_PREFIX.length());
        // lookup the type
        Type type = functionAndTypeManager.getType(parseTypeSignature(typeName));
        // verify we have one parameter of the proper type
        checkArgument(parameterTypes.size() == 1, "Expected one argument to literal function, but got %s", parameterTypes);
        Type parameterType = functionAndTypeManager.getType(parameterTypes.get(0));
        requireNonNull(parameterType, format("Type %s not found", parameterTypes.get(0)));
        return new SpecializedFunctionKey(magicLiteralFunction, BoundVariables.builder().setTypeVariable("T", parameterType).setTypeVariable("R", type).build(), 1);
    }
    throw new PrestoException(FUNCTION_IMPLEMENTATION_MISSING, format("%s not found", signature));
}

Example 4 with TypeSignature

use of com.facebook.presto.common.type.TypeSignature in project presto by prestodb.

the class TestAnnotationEngineForAggregates method testSimpleExplicitSpecializedAggregationParse.

// @Test - this is not yet supported
public void testSimpleExplicitSpecializedAggregationParse() {
    Signature expectedSignature = new Signature(QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "explicit_specialized_aggregate"), FunctionKind.AGGREGATE, ImmutableList.of(typeVariable("T")), ImmutableList.of(), parseTypeSignature("T"), ImmutableList.of(new TypeSignature(ARRAY, TypeSignatureParameter.of(parseTypeSignature("T")))), false);
    ParametricAggregation aggregation = parseFunctionDefinition(ExplicitSpecializedAggregationFunction.class);
    assertEquals(aggregation.getDescription(), "Simple explicit specialized aggregate");
    assertTrue(aggregation.isDeterministic());
    assertEquals(aggregation.getSignature(), expectedSignature);
    ParametricImplementationsGroup<AggregationImplementation> implementations = aggregation.getImplementations();
    assertImplementationCount(implementations, 0, 1, 1);
    AggregationImplementation implementation1 = implementations.getSpecializedImplementations().get(0);
    assertTrue(implementation1.hasSpecializedTypeParameters());
    assertFalse(implementation1.hasSpecializedTypeParameters());
    List<AggregationMetadata.ParameterMetadata.ParameterType> expectedMetadataTypes = ImmutableList.of(AggregationMetadata.ParameterMetadata.ParameterType.STATE, AggregationMetadata.ParameterMetadata.ParameterType.INPUT_CHANNEL);
    assertTrue(implementation1.getInputParameterMetadataTypes().equals(expectedMetadataTypes));
    AggregationImplementation implementation2 = implementations.getSpecializedImplementations().get(1);
    assertTrue(implementation2.hasSpecializedTypeParameters());
    assertFalse(implementation2.hasSpecializedTypeParameters());
    assertTrue(implementation2.getInputParameterMetadataTypes().equals(expectedMetadataTypes));
    InternalAggregationFunction specialized = aggregation.specialize(BoundVariables.builder().setTypeVariable("T", DoubleType.DOUBLE).build(), 1, FUNCTION_AND_TYPE_MANAGER);
    assertEquals(specialized.getFinalType(), DoubleType.DOUBLE);
    assertTrue(specialized.isDecomposable());
    assertEquals(specialized.name(), "implicit_specialized_aggregate");
}

Example 5 with TypeSignature

use of com.facebook.presto.common.type.TypeSignature in project presto by prestodb.

the class TestSignatureBinder method testBindLiteralForRepeatedVarchar.

@Test
public void testBindLiteralForRepeatedVarchar() {
    Set<String> literalParameters = ImmutableSet.of("x");
    TypeSignature leftType = parseTypeSignature("varchar(x)", literalParameters);
    TypeSignature rightType = parseTypeSignature("varchar(x)", literalParameters);
    TypeSignature returnType = parseTypeSignature("varchar(x)", literalParameters);
    Signature function = functionSignature().returnType(returnType).argumentTypes(leftType, rightType).build();
    assertThat(function).withCoercion().boundTo(ImmutableList.of("varchar(3)", "varchar(5)"), "varchar(5)").produces(new BoundVariables(ImmutableMap.of(), ImmutableMap.of("x", 5L)));
    assertThat(function).withCoercion().boundTo(ImmutableList.of("varchar(3)", "varchar(5)"), "varchar(6)").produces(new BoundVariables(ImmutableMap.of(), ImmutableMap.of("x", 6L)));
}