Examples with VARCHAR io.trino.spi.type.VarcharType.VARCHAR used on opensource projects

Example 31 with VARCHAR

use of io.trino.spi.type.VarcharType.VARCHAR in project trino by trinodb.

the class TestLocalExchange method testPartitionCustomPartitioning.

@Test(dataProvider = "executionStrategy")
public void testPartitionCustomPartitioning(PipelineExecutionStrategy executionStrategy) {
    ConnectorPartitioningHandle connectorPartitioningHandle = new ConnectorPartitioningHandle() {
    };
    ConnectorNodePartitioningProvider connectorNodePartitioningProvider = new ConnectorNodePartitioningProvider() {

        @Override
        public ConnectorBucketNodeMap getBucketNodeMap(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorPartitioningHandle partitioningHandle) {
            return createBucketNodeMap(2);
        }

        @Override
        public ToIntFunction<ConnectorSplit> getSplitBucketFunction(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorPartitioningHandle partitioningHandle) {
            throw new UnsupportedOperationException();
        }

        @Override
        public BucketFunction getBucketFunction(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorPartitioningHandle partitioningHandle, List<Type> partitionChannelTypes, int bucketCount) {
            return (page, position) -> {
                long rowValue = BIGINT.getLong(page.getBlock(0), position);
                if (rowValue == 42) {
                    return 0;
                }
                return 1;
            };
        }
    };
    List<Type> types = ImmutableList.of(VARCHAR, BIGINT);
    nodePartitioningManager.addPartitioningProvider(new CatalogName("foo"), connectorNodePartitioningProvider);
    PartitioningHandle partitioningHandle = new PartitioningHandle(Optional.of(new CatalogName("foo")), Optional.of(TestingTransactionHandle.create()), connectorPartitioningHandle);
    LocalExchangeFactory localExchangeFactory = new LocalExchangeFactory(nodePartitioningManager, SESSION, partitioningHandle, 2, types, ImmutableList.of(1), Optional.empty(), executionStrategy, LOCAL_EXCHANGE_MAX_BUFFERED_BYTES, TYPE_OPERATOR_FACTORY);
    LocalExchangeSinkFactoryId localExchangeSinkFactoryId = localExchangeFactory.newSinkFactoryId();
    localExchangeFactory.noMoreSinkFactories();
    run(localExchangeFactory, executionStrategy, exchange -> {
        assertEquals(exchange.getBufferCount(), 2);
        assertExchangeTotalBufferedBytes(exchange, 0);
        LocalExchangeSinkFactory sinkFactory = exchange.getSinkFactory(localExchangeSinkFactoryId);
        LocalExchangeSink sink = sinkFactory.createSink();
        assertSinkCanWrite(sink);
        sinkFactory.close();
        sinkFactory.noMoreSinkFactories();
        LocalExchangeSource sourceA = exchange.getSource(1);
        assertSource(sourceA, 0);
        LocalExchangeSource sourceB = exchange.getSource(0);
        assertSource(sourceB, 0);
        Page pageA = SequencePageBuilder.createSequencePage(types, 1, 100, 42);
        sink.addPage(pageA);
        assertSource(sourceA, 1);
        assertSource(sourceB, 0);
        assertRemovePage(types, sourceA, pageA);
        assertSource(sourceA, 0);
        Page pageB = SequencePageBuilder.createSequencePage(types, 100, 100, 43);
        sink.addPage(pageB);
        assertSource(sourceA, 0);
        assertSource(sourceB, 1);
        assertRemovePage(types, sourceB, pageB);
        assertSource(sourceB, 0);
    });
}

Example 32 with VARCHAR

use of io.trino.spi.type.VarcharType.VARCHAR in project trino by trinodb.

the class TestCachingHiveMetastore method testLoadAfterInvalidate.

@Test(timeOut = 60_000, dataProviderClass = DataProviders.class, dataProvider = "trueFalse")
public void testLoadAfterInvalidate(boolean invalidateAll) throws Exception {
    // State
    CopyOnWriteArrayList<Column> tableColumns = new CopyOnWriteArrayList<>();
    ConcurrentMap<String, Partition> tablePartitionsByName = new ConcurrentHashMap<>();
    Map<String, String> tableParameters = new ConcurrentHashMap<>();
    tableParameters.put("frequent-changing-table-parameter", "parameter initial value");
    // Initialize data
    String databaseName = "my_database";
    String tableName = "my_table_name";
    tableColumns.add(new Column("value", toHiveType(VARCHAR), Optional.empty()));
    tableColumns.add(new Column("pk", toHiveType(VARCHAR), Optional.empty()));
    List<String> partitionNames = new ArrayList<>();
    for (int i = 0; i < 10; i++) {
        String partitionName = "pk=" + i;
        tablePartitionsByName.put(partitionName, Partition.builder().setDatabaseName(databaseName).setTableName(tableName).setColumns(ImmutableList.copyOf(tableColumns)).setValues(List.of(Integer.toString(i))).withStorage(storage -> storage.setStorageFormat(fromHiveStorageFormat(TEXTFILE))).setParameters(Map.of("frequent-changing-partition-parameter", "parameter initial value")).build());
        partitionNames.add(partitionName);
    }
    // Mock metastore
    CountDownLatch getTableEnteredLatch = new CountDownLatch(1);
    CountDownLatch getTableReturnLatch = new CountDownLatch(1);
    CountDownLatch getTableFinishedLatch = new CountDownLatch(1);
    CountDownLatch getPartitionsByNamesEnteredLatch = new CountDownLatch(1);
    CountDownLatch getPartitionsByNamesReturnLatch = new CountDownLatch(1);
    CountDownLatch getPartitionsByNamesFinishedLatch = new CountDownLatch(1);
    HiveMetastore mockMetastore = new UnimplementedHiveMetastore() {

        @Override
        public Optional<Table> getTable(String databaseName, String tableName) {
            Optional<Table> table = Optional.of(Table.builder().setDatabaseName(databaseName).setTableName(tableName).setTableType(EXTERNAL_TABLE.name()).setDataColumns(tableColumns).setParameters(ImmutableMap.copyOf(tableParameters)).withStorage(storage -> storage.setStorageFormat(fromHiveStorageFormat(TEXTFILE))).setOwner(Optional.empty()).build());
            // 1
            getTableEnteredLatch.countDown();
            // 2
            await(getTableReturnLatch, 10, SECONDS);
            return table;
        }

        @Override
        public Map<String, Optional<Partition>> getPartitionsByNames(Table table, List<String> partitionNames) {
            Map<String, Optional<Partition>> result = new HashMap<>();
            for (String partitionName : partitionNames) {
                result.put(partitionName, Optional.ofNullable(tablePartitionsByName.get(partitionName)));
            }
            // loader#1
            getPartitionsByNamesEnteredLatch.countDown();
            // loader#2
            await(getPartitionsByNamesReturnLatch, 10, SECONDS);
            return result;
        }
    };
    // Caching metastore
    metastore = cachingHiveMetastore(mockMetastore, executor, new Duration(5, TimeUnit.MINUTES), Optional.of(new Duration(1, TimeUnit.MINUTES)), 1000);
    // The test. Main thread does modifications and verifies subsequent load sees them. Background thread loads the state into the cache.
    ExecutorService executor = Executors.newFixedThreadPool(1);
    try {
        Future<Void> future = executor.submit(() -> {
            try {
                Table table;
                table = metastore.getTable(databaseName, tableName).orElseThrow();
                // 3
                getTableFinishedLatch.countDown();
                metastore.getPartitionsByNames(table, partitionNames);
                // 6
                getPartitionsByNamesFinishedLatch.countDown();
                return (Void) null;
            } catch (Throwable e) {
                log.error(e);
                throw e;
            }
        });
        // 21
        await(getTableEnteredLatch, 10, SECONDS);
        tableParameters.put("frequent-changing-table-parameter", "main-thread-put-xyz");
        if (invalidateAll) {
            metastore.flushCache();
        } else {
            metastore.invalidateTable(databaseName, tableName);
        }
        // 2
        getTableReturnLatch.countDown();
        // 3
        await(getTableFinishedLatch, 10, SECONDS);
        Table table = metastore.getTable(databaseName, tableName).orElseThrow();
        assertThat(table.getParameters()).isEqualTo(Map.of("frequent-changing-table-parameter", "main-thread-put-xyz"));
        // 4
        await(getPartitionsByNamesEnteredLatch, 10, SECONDS);
        String partitionName = partitionNames.get(2);
        Map<String, String> newPartitionParameters = Map.of("frequent-changing-partition-parameter", "main-thread-put-alice");
        tablePartitionsByName.put(partitionName, Partition.builder(tablePartitionsByName.get(partitionName)).setParameters(newPartitionParameters).build());
        if (invalidateAll) {
            metastore.flushCache();
        } else {
            metastore.invalidateTable(databaseName, tableName);
        }
        // 5
        getPartitionsByNamesReturnLatch.countDown();
        // 6
        await(getPartitionsByNamesFinishedLatch, 10, SECONDS);
        Map<String, Optional<Partition>> loadedPartitions = metastore.getPartitionsByNames(table, partitionNames);
        assertThat(loadedPartitions.get(partitionName)).isNotNull().isPresent().hasValueSatisfying(partition -> assertThat(partition.getParameters()).isEqualTo(newPartitionParameters));
        // verify no failure in the background thread
        future.get(10, SECONDS);
    } finally {
        getTableEnteredLatch.countDown();
        getTableReturnLatch.countDown();
        getTableFinishedLatch.countDown();
        getPartitionsByNamesEnteredLatch.countDown();
        getPartitionsByNamesReturnLatch.countDown();
        getPartitionsByNamesFinishedLatch.countDown();
        executor.shutdownNow();
        executor.awaitTermination(10, SECONDS);
    }
}

Example 33 with VARCHAR

use of io.trino.spi.type.VarcharType.VARCHAR in project trino by trinodb.

the class QueryPlanner method planExpand.

public RelationPlan planExpand(Query query) {
    checkArgument(analysis.isExpandableQuery(query), "query is not registered as expandable");
    Union union = (Union) query.getQueryBody();
    ImmutableList.Builder<NodeAndMappings> recursionSteps = ImmutableList.builder();
    // plan anchor relation
    Relation anchorNode = union.getRelations().get(0);
    RelationPlan anchorPlan = new RelationPlanner(analysis, symbolAllocator, idAllocator, lambdaDeclarationToSymbolMap, plannerContext, outerContext, session, recursiveSubqueries).process(anchorNode, null);
    // prune anchor plan outputs to contain only the symbols exposed in the scope
    NodeAndMappings prunedAnchorPlan = pruneInvisibleFields(anchorPlan, idAllocator);
    // if the anchor plan has duplicate output symbols, add projection on top to make the symbols unique
    // This is necessary to successfully unroll recursion: the recursion step relation must follow
    // the same layout while it might not have duplicate outputs where the anchor plan did
    NodeAndMappings disambiguatedAnchorPlan = disambiguateOutputs(prunedAnchorPlan, symbolAllocator, idAllocator);
    anchorPlan = new RelationPlan(disambiguatedAnchorPlan.getNode(), analysis.getScope(query), disambiguatedAnchorPlan.getFields(), outerContext);
    recursionSteps.add(copy(anchorPlan.getRoot(), anchorPlan.getFieldMappings()));
    // plan recursion step
    Relation recursionStepRelation = union.getRelations().get(1);
    RelationPlan recursionStepPlan = new RelationPlanner(analysis, symbolAllocator, idAllocator, lambdaDeclarationToSymbolMap, plannerContext, outerContext, session, ImmutableMap.of(NodeRef.of(analysis.getRecursiveReference(query)), anchorPlan)).process(recursionStepRelation, null);
    // coerce recursion step outputs and prune them to contain only the symbols exposed in the scope
    NodeAndMappings coercedRecursionStep;
    List<Type> types = analysis.getRelationCoercion(recursionStepRelation);
    if (types == null) {
        coercedRecursionStep = pruneInvisibleFields(recursionStepPlan, idAllocator);
    } else {
        coercedRecursionStep = coerce(recursionStepPlan, types, symbolAllocator, idAllocator);
    }
    NodeAndMappings replacementSpot = new NodeAndMappings(anchorPlan.getRoot(), anchorPlan.getFieldMappings());
    PlanNode recursionStep = coercedRecursionStep.getNode();
    List<Symbol> mappings = coercedRecursionStep.getFields();
    // unroll recursion
    int maxRecursionDepth = getMaxRecursionDepth(session);
    for (int i = 0; i < maxRecursionDepth; i++) {
        recursionSteps.add(copy(recursionStep, mappings));
        NodeAndMappings replacement = copy(recursionStep, mappings);
        // if the recursion step plan has duplicate output symbols, add projection on top to make the symbols unique
        // This is necessary to successfully unroll recursion: the relation on the next recursion step must follow
        // the same layout while it might not have duplicate outputs where the plan for this step did
        replacement = disambiguateOutputs(replacement, symbolAllocator, idAllocator);
        recursionStep = replace(recursionStep, replacementSpot, replacement);
        replacementSpot = replacement;
    }
    // after the last recursion step, check if the recursion converged. the last step is expected to return empty result
    // 1. append window to count rows
    NodeAndMappings checkConvergenceStep = copy(recursionStep, mappings);
    Symbol countSymbol = symbolAllocator.newSymbol("count", BIGINT);
    ResolvedFunction function = plannerContext.getMetadata().resolveFunction(session, QualifiedName.of("count"), ImmutableList.of());
    WindowNode.Function countFunction = new WindowNode.Function(function, ImmutableList.of(), DEFAULT_FRAME, false);
    WindowNode windowNode = new WindowNode(idAllocator.getNextId(), checkConvergenceStep.getNode(), new WindowNode.Specification(ImmutableList.of(), Optional.empty()), ImmutableMap.of(countSymbol, countFunction), Optional.empty(), ImmutableSet.of(), 0);
    // 2. append filter to fail on non-empty result
    ResolvedFunction fail = plannerContext.getMetadata().resolveFunction(session, QualifiedName.of("fail"), fromTypes(VARCHAR));
    String recursionLimitExceededMessage = format("Recursion depth limit exceeded (%s). Use 'max_recursion_depth' session property to modify the limit.", maxRecursionDepth);
    Expression predicate = new IfExpression(new ComparisonExpression(GREATER_THAN_OR_EQUAL, countSymbol.toSymbolReference(), new GenericLiteral("BIGINT", "0")), new Cast(new FunctionCall(fail.toQualifiedName(), ImmutableList.of(new Cast(new StringLiteral(recursionLimitExceededMessage), toSqlType(VARCHAR)))), toSqlType(BOOLEAN)), TRUE_LITERAL);
    FilterNode filterNode = new FilterNode(idAllocator.getNextId(), windowNode, predicate);
    recursionSteps.add(new NodeAndMappings(filterNode, checkConvergenceStep.getFields()));
    // union all the recursion steps
    List<NodeAndMappings> recursionStepsToUnion = recursionSteps.build();
    List<Symbol> unionOutputSymbols = anchorPlan.getFieldMappings().stream().map(symbol -> symbolAllocator.newSymbol(symbol, "_expanded")).collect(toImmutableList());
    ImmutableListMultimap.Builder<Symbol, Symbol> unionSymbolMapping = ImmutableListMultimap.builder();
    for (NodeAndMappings plan : recursionStepsToUnion) {
        for (int i = 0; i < unionOutputSymbols.size(); i++) {
            unionSymbolMapping.put(unionOutputSymbols.get(i), plan.getFields().get(i));
        }
    }
    List<PlanNode> nodesToUnion = recursionStepsToUnion.stream().map(NodeAndMappings::getNode).collect(toImmutableList());
    PlanNode result = new UnionNode(idAllocator.getNextId(), nodesToUnion, unionSymbolMapping.build(), unionOutputSymbols);
    if (union.isDistinct()) {
        result = new AggregationNode(idAllocator.getNextId(), result, ImmutableMap.of(), singleGroupingSet(result.getOutputSymbols()), ImmutableList.of(), AggregationNode.Step.SINGLE, Optional.empty(), Optional.empty());
    }
    return new RelationPlan(result, anchorPlan.getScope(), unionOutputSymbols, outerContext);
}

Example 34 with VARCHAR

use of io.trino.spi.type.VarcharType.VARCHAR in project trino by trinodb.

the class TestPushProjectionIntoTableScan method testPushProjection.

@Test
public void testPushProjection() {
    try (RuleTester ruleTester = defaultRuleTester()) {
        // Building context for input
        String columnName = "col0";
        Type columnType = ROW_TYPE;
        Symbol baseColumn = new Symbol(columnName);
        ColumnHandle columnHandle = new TpchColumnHandle(columnName, columnType);
        // Create catalog with applyProjection enabled
        MockConnectorFactory factory = createMockFactory(ImmutableMap.of(columnName, columnHandle), Optional.of(this::mockApplyProjection));
        ruleTester.getQueryRunner().createCatalog(MOCK_CATALOG, factory, ImmutableMap.of());
        TypeAnalyzer typeAnalyzer = createTestingTypeAnalyzer(ruleTester.getPlannerContext());
        // Prepare project node symbols and types
        Symbol identity = new Symbol("symbol_identity");
        Symbol dereference = new Symbol("symbol_dereference");
        Symbol constant = new Symbol("symbol_constant");
        Symbol call = new Symbol("symbol_call");
        ImmutableMap<Symbol, Type> types = ImmutableMap.of(baseColumn, ROW_TYPE, identity, ROW_TYPE, dereference, BIGINT, constant, BIGINT, call, VARCHAR);
        // Prepare project node assignments
        ImmutableMap<Symbol, Expression> inputProjections = ImmutableMap.of(identity, baseColumn.toSymbolReference(), dereference, new SubscriptExpression(baseColumn.toSymbolReference(), new LongLiteral("1")), constant, new LongLiteral("5"), call, new FunctionCall(QualifiedName.of("STARTS_WITH"), ImmutableList.of(new StringLiteral("abc"), new StringLiteral("ab"))));
        // Compute expected symbols after applyProjection
        TransactionId transactionId = ruleTester.getQueryRunner().getTransactionManager().beginTransaction(false);
        Session session = MOCK_SESSION.beginTransactionId(transactionId, ruleTester.getQueryRunner().getTransactionManager(), ruleTester.getQueryRunner().getAccessControl());
        ImmutableMap<Symbol, String> connectorNames = inputProjections.entrySet().stream().collect(toImmutableMap(Map.Entry::getKey, e -> translate(session, e.getValue(), typeAnalyzer, viewOf(types), ruleTester.getPlannerContext()).get().toString()));
        ImmutableMap<Symbol, String> newNames = ImmutableMap.of(identity, "projected_variable_" + connectorNames.get(identity), dereference, "projected_dereference_" + connectorNames.get(dereference), constant, "projected_constant_" + connectorNames.get(constant), call, "projected_call_" + connectorNames.get(call));
        Map<String, ColumnHandle> expectedColumns = newNames.entrySet().stream().collect(toImmutableMap(Map.Entry::getValue, e -> column(e.getValue(), types.get(e.getKey()))));
        ruleTester.assertThat(createRule(ruleTester)).on(p -> {
            // Register symbols
            types.forEach((symbol, type) -> p.symbol(symbol.getName(), type));
            return p.project(new Assignments(inputProjections), p.tableScan(tableScan -> tableScan.setTableHandle(TEST_TABLE_HANDLE).setSymbols(ImmutableList.copyOf(types.keySet())).setAssignments(types.keySet().stream().collect(Collectors.toMap(Function.identity(), v -> columnHandle))).setStatistics(Optional.of(PlanNodeStatsEstimate.builder().setOutputRowCount(42).addSymbolStatistics(baseColumn, SymbolStatsEstimate.builder().setNullsFraction(0).setDistinctValuesCount(33).build()).build()))));
        }).withSession(MOCK_SESSION).matches(project(newNames.entrySet().stream().collect(toImmutableMap(e -> e.getKey().getName(), e -> expression(symbolReference(e.getValue())))), tableScan(new MockConnectorTableHandle(new SchemaTableName(TEST_SCHEMA, "projected_" + TEST_TABLE), TupleDomain.all(), Optional.of(ImmutableList.copyOf(expectedColumns.values())))::equals, TupleDomain.all(), expectedColumns.entrySet().stream().collect(toImmutableMap(Map.Entry::getKey, e -> e.getValue()::equals)), Optional.of(PlanNodeStatsEstimate.builder().setOutputRowCount(42).addSymbolStatistics(new Symbol(newNames.get(constant)), SymbolStatsEstimate.builder().setDistinctValuesCount(1).setNullsFraction(0).setLowValue(5).setHighValue(5).build()).addSymbolStatistics(new Symbol(newNames.get(call).toLowerCase(ENGLISH)), SymbolStatsEstimate.builder().setDistinctValuesCount(1).setNullsFraction(0).build()).addSymbolStatistics(new Symbol(newNames.get(identity)), SymbolStatsEstimate.builder().setDistinctValuesCount(33).setNullsFraction(0).build()).addSymbolStatistics(new Symbol(newNames.get(dereference)), SymbolStatsEstimate.unknown()).build())::equals)));
    }
}

Example 35 with VARCHAR

use of io.trino.spi.type.VarcharType.VARCHAR in project trino by trinodb.

the class TestPushTopNIntoTableScan method testPushPartialTopNIntoTableScanNotGuaranteed.

@Test
public void testPushPartialTopNIntoTableScanNotGuaranteed() {
    try (RuleTester ruleTester = defaultRuleTester()) {
        MockConnectorTableHandle connectorHandle = new MockConnectorTableHandle(TEST_SCHEMA_TABLE);
        // make the mock connector return a new connectorHandle
        MockConnectorFactory.ApplyTopN applyTopN = (session, handle, topNCount, sortItems, tableAssignments) -> Optional.of(new TopNApplicationResult<>(connectorHandle, false, false));
        MockConnectorFactory mockFactory = createMockFactory(assignments, Optional.of(applyTopN));
        ruleTester.getQueryRunner().createCatalog(MOCK_CATALOG, mockFactory, ImmutableMap.of());
        ruleTester.assertThat(new PushTopNIntoTableScan(ruleTester.getMetadata())).on(p -> {
            Symbol dimension = p.symbol(dimensionName, VARCHAR);
            Symbol metric = p.symbol(metricName, BIGINT);
            return p.topN(1, ImmutableList.of(dimension), TopNNode.Step.PARTIAL, p.tableScan(TEST_TABLE_HANDLE, ImmutableList.of(dimension, metric), ImmutableMap.of(dimension, dimensionColumn, metric, metricColumn)));
        }).withSession(MOCK_SESSION).matches(topN(1, ImmutableList.of(sort(dimensionName, ASCENDING, FIRST)), TopNNode.Step.PARTIAL, tableScan(connectorHandle::equals, TupleDomain.all(), ImmutableMap.of(dimensionName, dimensionColumn::equals, metricName, metricColumn::equals))));
    }
}