Examples with FunctionAndTypeManager com.facebook.presto.metadata.FunctionAndTypeManager used on opensource projects

Example 56 with FunctionAndTypeManager

use of com.facebook.presto.metadata.FunctionAndTypeManager in project presto by prestodb.

the class TestIllegalMethodAggregation method testNullMethod.

@Test
public void testNullMethod() {
    FunctionAndTypeManager functionAndTypeManager = MetadataManager.createTestMetadataManager().getFunctionAndTypeManager();
    InternalAggregationFunction function = functionAndTypeManager.getAggregateFunctionImplementation(functionAndTypeManager.lookupFunction("differential_entropy", fromTypes(BIGINT, DOUBLE, DOUBLE, VARCHAR, DOUBLE, DOUBLE)));
    createStringsBlock((String) null);
    aggregation(function, createLongsBlock(200), createDoublesBlock(0.1), createDoublesBlock(-0.2), createStringsBlock((String) null), createDoublesBlock(0.0), createDoublesBlock(1.0));
}

Example 57 with FunctionAndTypeManager

use of com.facebook.presto.metadata.FunctionAndTypeManager in project presto by prestodb.

the class OrcTestingUtil method createRecordReader.

public static OrcBatchRecordReader createRecordReader(OrcReader orcReader, Map<Integer, Type> includedColumns) throws OrcCorruptionException {
    Metadata metadata = MetadataManager.createTestMetadataManager();
    FunctionAndTypeManager functionAndTypeManager = metadata.getFunctionAndTypeManager();
    StorageTypeConverter storageTypeConverter = new StorageTypeConverter(functionAndTypeManager);
    return orcReader.createBatchRecordReader(storageTypeConverter.toStorageTypes(includedColumns), OrcPredicate.TRUE, DateTimeZone.UTC, new RaptorOrcAggregatedMemoryContext(), MAX_BATCH_SIZE);
}

Example 58 with FunctionAndTypeManager

use of com.facebook.presto.metadata.FunctionAndTypeManager in project presto by prestodb.

the class TestBucketBalancer method setup.

@BeforeMethod
public void setup() {
    FunctionAndTypeManager functionAndTypeManager = createTestFunctionAndTypeManager();
    dbi = new DBI("jdbc:h2:mem:test" + System.nanoTime() + "_" + ThreadLocalRandom.current().nextInt());
    dbi.registerMapper(new Distribution.Mapper(functionAndTypeManager));
    dummyHandle = dbi.open();
    createTablesWithRetry(dbi);
    metadataDao = dbi.onDemand(MetadataDao.class);
    nodeManager = new TestingNodeManager(AVAILABLE_WORKERS.stream().map(TestBucketBalancer::createTestingNode).collect(Collectors.toList()));
    NodeSupplier nodeSupplier = nodeManager::getWorkerNodes;
    shardManager = createShardManager(dbi, nodeSupplier);
    balancer = new BucketBalancer(nodeSupplier, shardManager, true, new Duration(1, DAYS), 0, true, true, "test");
}

Example 59 with FunctionAndTypeManager

use of com.facebook.presto.metadata.FunctionAndTypeManager in project presto by prestodb.

the class TestOrcFileRewriter method testRewriterDropThenAddSameColumns.

/**
 * The following test drop and add the same columns; the legacy ORC rewriter will fail due to unchanged schema.
 * However, if we enforce the newly added column to always have the largest ID, this won't happen.
 */
@Test
public void testRewriterDropThenAddSameColumns() throws Exception {
    FunctionAndTypeManager functionAndTypeManager = createTestFunctionAndTypeManager();
    DBI dbi = new DBI("jdbc:h2:mem:test" + System.nanoTime() + "_" + ThreadLocalRandom.current().nextInt());
    dbi.registerMapper(new TableColumn.Mapper(functionAndTypeManager));
    Handle dummyHandle = dbi.open();
    File dataDir = Files.createTempDir();
    StorageManager storageManager = createOrcStorageManager(dbi, dataDir);
    List<Long> columnIds = ImmutableList.of(3L, 7L);
    List<Type> columnTypes = ImmutableList.of(BIGINT, createVarcharType(20));
    File file = new File(temporary, randomUUID().toString());
    try (FileWriter writer = createFileWriter(columnIds, columnTypes, file, false)) {
        List<Page> pages = rowPagesBuilder(columnTypes).row(2L, "2").build();
        writer.appendPages(pages);
    }
    // Add a column
    File newFile1 = new File(temporary, randomUUID().toString());
    FileSystem fileSystem = new LocalOrcDataEnvironment().getFileSystem(DEFAULT_RAPTOR_CONTEXT);
    OrcFileInfo info = createFileRewriter().rewrite(fileSystem, getColumnTypes(ImmutableList.of(3L, 7L, 10L), ImmutableList.of(BIGINT, createVarcharType(20), DOUBLE)), path(file), path(newFile1), new BitSet(5));
    assertEquals(info.getRowCount(), 1);
    // Drop a column
    File newFile2 = new File(temporary, randomUUID().toString());
    info = createFileRewriter().rewrite(fileSystem, getColumnTypes(ImmutableList.of(7L, 10L), ImmutableList.of(createVarcharType(20), DOUBLE)), path(newFile1), path(newFile2), new BitSet(5));
    assertEquals(info.getRowCount(), 1);
    // Add a column with the same ID but different type
    File newFile3 = new File(temporary, randomUUID().toString());
    info = createFileRewriter().rewrite(fileSystem, getColumnTypes(ImmutableList.of(7L, 10L, 3L), ImmutableList.of(createVarcharType(20), DOUBLE, createVarcharType(5))), path(newFile2), path(newFile3), new BitSet(5));
    assertEquals(info.getRowCount(), 1);
    // Get prepared for the final file; make sure it is accessible from storage manager
    UUID uuid = randomUUID();
    File newFile4 = getFileSystemPath(new File(dataDir, "data/storage"), uuid);
    // Optimized ORC writer does not create the file itself
    newFile4.getParentFile().mkdirs();
    newFile4.createNewFile();
    // Drop a column and add a column
    info = createFileRewriter().rewrite(fileSystem, getColumnTypes(ImmutableList.of(7L, 3L, 8L), ImmutableList.of(createVarcharType(20), createVarcharType(5), INTEGER)), path(newFile3), path(newFile4), new BitSet(5));
    assertEquals(info.getRowCount(), 1);
    ConnectorPageSource source = storageManager.getPageSource(DEFAULT_RAPTOR_CONTEXT, DEFAULT_HIVE_FILE_CONTEXT, uuid, Optional.empty(), false, OptionalInt.empty(), ImmutableList.of(3L, 7L, 8L), ImmutableList.of(createVarcharType(5), createVarcharType(20), INTEGER), TupleDomain.all(), READER_ATTRIBUTES);
    Page page = null;
    while (page == null) {
        page = source.getNextPage();
    }
    assertEquals(page.getPositionCount(), 1);
    try {
        // Column 3L
        Block column0 = page.getBlock(0);
        assertTrue(column0.isNull(0));
        // Column 7L
        Block column1 = page.getBlock(1);
        assertEquals(createVarcharType(20).getSlice(column1, 0), utf8Slice("2"));
        // Column 8L
        Block column2 = page.getBlock(2);
        assertTrue(column2.isNull(0));
        dummyHandle.close();
        deleteRecursively(dataDir.toPath(), ALLOW_INSECURE);
    } catch (UnsupportedOperationException e) {
        // Optimized ORC rewriter will respect the schema
        fail();
    }
}

Example 60 with FunctionAndTypeManager

use of com.facebook.presto.metadata.FunctionAndTypeManager in project presto by prestodb.

the class TestShardMetadataRecordCursor method setup.

@BeforeMethod
public void setup() {
    FunctionAndTypeManager functionAndTypeManager = createTestFunctionAndTypeManager();
    this.dbi = new DBI("jdbc:h2:mem:test" + System.nanoTime() + "_" + ThreadLocalRandom.current().nextInt());
    this.dbi.registerMapper(new TableColumn.Mapper(functionAndTypeManager));
    this.dummyHandle = dbi.open();
    createTablesWithRetry(dbi);
    this.metadata = new RaptorMetadata("raptor", dbi, createShardManager(dbi), functionAndTypeManager);
    // Create table
    ConnectorTableMetadata table = tableMetadataBuilder(DEFAULT_TEST_ORDERS).column("orderkey", BIGINT).column("orderdate", DATE).property("temporal_column", "orderdate").property(TABLE_SUPPORTS_DELTA_DELETE, false).build();
    createTable(table);
}