Examples with TypeOperators io.trino.spi.type.TypeOperators used on opensource projects

Example 16 with TypeOperators

use of io.trino.spi.type.TypeOperators in project trino by trinodb.

the class TestShardCompactor method setup.

@BeforeMethod
public void setup() throws IOException {
    temporary = createTempDirectory(null);
    Jdbi dbi = createTestingJdbi();
    dummyHandle = dbi.open();
    storageManager = createRaptorStorageManager(dbi, temporary.toFile(), MAX_SHARD_ROWS);
    compactor = new ShardCompactor(storageManager, READER_OPTIONS, new TypeOperators());
}

Example 17 with TypeOperators

use of io.trino.spi.type.TypeOperators in project trino by trinodb.

the class HashBuildAndJoinBenchmark method createDrivers.

/*
    select orderkey, quantity, totalprice
    from lineitem join orders using (orderkey)
     */
@Override
protected List<Driver> createDrivers(TaskContext taskContext) {
    ImmutableList.Builder<OperatorFactory> driversBuilder = ImmutableList.builder();
    driversBuilder.add(ordersTableScan);
    List<Type> sourceTypes = ordersTableTypes;
    OptionalInt hashChannel = OptionalInt.empty();
    if (hashEnabled) {
        driversBuilder.add(createHashProjectOperator(1, new PlanNodeId("test"), sourceTypes));
        sourceTypes = ImmutableList.<Type>builder().addAll(sourceTypes).add(BIGINT).build();
        hashChannel = OptionalInt.of(sourceTypes.size() - 1);
    }
    // hash build
    BlockTypeOperators blockTypeOperators = new BlockTypeOperators(new TypeOperators());
    JoinBridgeManager<PartitionedLookupSourceFactory> lookupSourceFactoryManager = JoinBridgeManager.lookupAllAtOnce(new PartitionedLookupSourceFactory(sourceTypes, ImmutableList.of(0, 1).stream().map(sourceTypes::get).collect(toImmutableList()), Ints.asList(0).stream().map(sourceTypes::get).collect(toImmutableList()), 1, false, blockTypeOperators));
    HashBuilderOperatorFactory hashBuilder = new HashBuilderOperatorFactory(2, new PlanNodeId("test"), lookupSourceFactoryManager, ImmutableList.of(0, 1), Ints.asList(0), hashChannel, Optional.empty(), Optional.empty(), ImmutableList.of(), 1_500_000, new PagesIndex.TestingFactory(false), false, SingleStreamSpillerFactory.unsupportedSingleStreamSpillerFactory(), incrementalLoadFactorHashArraySizeSupplier(session));
    driversBuilder.add(hashBuilder);
    DriverFactory hashBuildDriverFactory = new DriverFactory(0, true, false, driversBuilder.build(), OptionalInt.empty(), UNGROUPED_EXECUTION);
    // join
    ImmutableList.Builder<OperatorFactory> joinDriversBuilder = ImmutableList.builder();
    joinDriversBuilder.add(lineItemTableScan);
    sourceTypes = lineItemTableTypes;
    hashChannel = OptionalInt.empty();
    if (hashEnabled) {
        joinDriversBuilder.add(createHashProjectOperator(1, new PlanNodeId("test"), sourceTypes));
        sourceTypes = ImmutableList.<Type>builder().addAll(sourceTypes).add(BIGINT).build();
        hashChannel = OptionalInt.of(sourceTypes.size() - 1);
    }
    OperatorFactory joinOperator = operatorFactories.innerJoin(2, new PlanNodeId("test"), lookupSourceFactoryManager, false, false, false, sourceTypes, Ints.asList(0), hashChannel, Optional.empty(), OptionalInt.empty(), unsupportedPartitioningSpillerFactory(), blockTypeOperators);
    joinDriversBuilder.add(joinOperator);
    joinDriversBuilder.add(new NullOutputOperatorFactory(3, new PlanNodeId("test")));
    DriverFactory joinDriverFactory = new DriverFactory(1, true, true, joinDriversBuilder.build(), OptionalInt.empty(), UNGROUPED_EXECUTION);
    Driver hashBuildDriver = hashBuildDriverFactory.createDriver(taskContext.addPipelineContext(0, true, false, false).addDriverContext());
    hashBuildDriverFactory.noMoreDrivers();
    Driver joinDriver = joinDriverFactory.createDriver(taskContext.addPipelineContext(1, true, true, false).addDriverContext());
    joinDriverFactory.noMoreDrivers();
    return ImmutableList.of(hashBuildDriver, joinDriver);
}

Example 18 with TypeOperators

use of io.trino.spi.type.TypeOperators in project yauaa by nielsbasjes.

the class ParseUserAgentFunction method parseUserAgent.

@ScalarFunction("parse_user_agent")
@Description("Tries to parse and analyze the provided useragent string and extract as many attributes " + "as possible. Uses Yauaa (Yet Another UserAgent Analyzer) version " + Version.PROJECT_VERSION + ". " + "See https://yauaa.basjes.nl/udf/trino/ for documentation.")
@SqlType("map(varchar, varchar)")
public static Block parseUserAgent(@SqlType(StandardTypes.VARCHAR) Slice userAgentSlice) throws IllegalArgumentException {
    String userAgentStringToParse = null;
    if (userAgentSlice != null) {
        userAgentStringToParse = userAgentSlice.toStringUtf8();
    }
    UserAgentAnalyzer userAgentAnalyzer = getInstance();
    UserAgent userAgent = userAgentAnalyzer.parse(userAgentStringToParse);
    Map<String, String> resultMap = userAgent.toMap(userAgent.getAvailableFieldNamesSorted());
    MapType mapType = new MapType(VARCHAR, VARCHAR, new TypeOperators());
    BlockBuilder blockBuilder = mapType.createBlockBuilder(null, resultMap.size());
    BlockBuilder singleMapBlockBuilder = blockBuilder.beginBlockEntry();
    for (Map.Entry<String, String> entry : resultMap.entrySet()) {
        VARCHAR.writeString(singleMapBlockBuilder, entry.getKey());
        VARCHAR.writeString(singleMapBlockBuilder, entry.getValue());
    }
    blockBuilder.closeEntry();
    return mapType.getObject(blockBuilder, blockBuilder.getPositionCount() - 1);
}

Example 19 with TypeOperators

use of io.trino.spi.type.TypeOperators in project yauaa by nielsbasjes.

the class TestParseFunction method testParser.

@Test
void testParser() {
    TreeMap<String, String> expected = new TreeMap<>();
    expected.put("DeviceClass", "Desktop");
    expected.put("DeviceName", "Linux Desktop");
    expected.put("DeviceBrand", "Unknown");
    expected.put("DeviceCpu", "Intel x86_64");
    expected.put("DeviceCpuBits", "64");
    expected.put("DeviceFirmwareVersion", "??");
    expected.put("DeviceVersion", "??");
    expected.put("OperatingSystemClass", "Desktop");
    expected.put("OperatingSystemName", "Linux");
    expected.put("OperatingSystemVersion", "??");
    expected.put("OperatingSystemVersionMajor", "??");
    expected.put("OperatingSystemNameVersion", "Linux ??");
    expected.put("OperatingSystemNameVersionMajor", "Linux ??");
    expected.put("OperatingSystemVersionBuild", "??");
    expected.put("LayoutEngineClass", "Browser");
    expected.put("LayoutEngineName", "Blink");
    expected.put("LayoutEngineVersion", "98.0");
    expected.put("LayoutEngineVersionMajor", "98");
    expected.put("LayoutEngineNameVersion", "Blink 98.0");
    expected.put("LayoutEngineNameVersionMajor", "Blink 98");
    expected.put("LayoutEngineBuild", "Unknown");
    expected.put("AgentClass", "Browser");
    expected.put("AgentName", "Chrome");
    expected.put("AgentVersion", "98.0.4758.102");
    expected.put("AgentVersionMajor", "98");
    expected.put("AgentNameVersion", "Chrome 98.0.4758.102");
    expected.put("AgentNameVersionMajor", "Chrome 98");
    expected.put("AgentBuild", "Unknown");
    expected.put("AgentLanguage", "Unknown");
    expected.put("AgentLanguageCode", "Unknown");
    expected.put("AgentInformationEmail", "Unknown");
    expected.put("AgentInformationUrl", "Unknown");
    expected.put("AgentSecurity", "Unknown");
    expected.put("AgentUuid", "Unknown");
    expected.put("WebviewAppName", "Unknown");
    expected.put("WebviewAppVersion", "??");
    expected.put("WebviewAppVersionMajor", "??");
    expected.put("WebviewAppNameVersion", "Unknown ??");
    expected.put("WebviewAppNameVersionMajor", "Unknown ??");
    expected.put("FacebookCarrier", "Unknown");
    expected.put("FacebookDeviceClass", "Unknown");
    expected.put("FacebookDeviceName", "Unknown");
    expected.put("FacebookDeviceVersion", "??");
    expected.put("FacebookFBOP", "Unknown");
    expected.put("FacebookFBSS", "Unknown");
    expected.put("FacebookOperatingSystemName", "Unknown");
    expected.put("FacebookOperatingSystemVersion", "??");
    expected.put("Anonymized", "Unknown");
    expected.put("HackerAttackVector", "Unknown");
    expected.put("HackerToolkit", "Unknown");
    expected.put("KoboAffiliate", "Unknown");
    expected.put("KoboPlatformId", "Unknown");
    expected.put("IECompatibilityVersion", "??");
    expected.put("IECompatibilityVersionMajor", "??");
    expected.put("IECompatibilityNameVersion", "Unknown ??");
    expected.put("IECompatibilityNameVersionMajor", "Unknown ??");
    expected.put("__SyntaxError__", "false");
    expected.put("Carrier", "Unknown");
    expected.put("GSAInstallationID", "Unknown");
    expected.put("NetworkType", "Unknown");
    assertFunction("parse_user_agent('Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36')", new MapType(VARCHAR, VARCHAR, new TypeOperators()), expected);
}

Example 20 with TypeOperators

use of io.trino.spi.type.TypeOperators in project trino by trinodb.

the class TestHttpRemoteTask method testDynamicFilters.

@Test(timeOut = 30000)
public void testDynamicFilters() throws Exception {
    DynamicFilterId filterId1 = new DynamicFilterId("df1");
    DynamicFilterId filterId2 = new DynamicFilterId("df2");
    SymbolAllocator symbolAllocator = new SymbolAllocator();
    Symbol symbol1 = symbolAllocator.newSymbol("DF_SYMBOL1", BIGINT);
    Symbol symbol2 = symbolAllocator.newSymbol("DF_SYMBOL2", BIGINT);
    SymbolReference df1 = symbol1.toSymbolReference();
    SymbolReference df2 = symbol2.toSymbolReference();
    ColumnHandle handle1 = new TestingColumnHandle("column1");
    ColumnHandle handle2 = new TestingColumnHandle("column2");
    QueryId queryId = new QueryId("test");
    TestingTaskResource testingTaskResource = new TestingTaskResource(new AtomicLong(System.nanoTime()), FailureScenario.NO_FAILURE);
    DynamicFilterService dynamicFilterService = new DynamicFilterService(PLANNER_CONTEXT.getMetadata(), PLANNER_CONTEXT.getFunctionManager(), new TypeOperators(), newDirectExecutorService());
    HttpRemoteTaskFactory httpRemoteTaskFactory = createHttpRemoteTaskFactory(testingTaskResource, dynamicFilterService);
    RemoteTask remoteTask = createRemoteTask(httpRemoteTaskFactory, ImmutableSet.of());
    Map<DynamicFilterId, Domain> initialDomain = ImmutableMap.of(filterId1, Domain.singleValue(BIGINT, 1L));
    testingTaskResource.setInitialTaskInfo(remoteTask.getTaskInfo());
    testingTaskResource.setDynamicFilterDomains(new VersionedDynamicFilterDomains(1L, initialDomain));
    dynamicFilterService.registerQuery(queryId, TEST_SESSION, ImmutableSet.of(filterId1, filterId2), ImmutableSet.of(filterId1, filterId2), ImmutableSet.of());
    dynamicFilterService.stageCannotScheduleMoreTasks(new StageId(queryId, 1), 0, 1);
    DynamicFilter dynamicFilter = dynamicFilterService.createDynamicFilter(queryId, ImmutableList.of(new DynamicFilters.Descriptor(filterId1, df1), new DynamicFilters.Descriptor(filterId2, df2)), ImmutableMap.of(symbol1, handle1, symbol2, handle2), symbolAllocator.getTypes());
    // make sure initial dynamic filters are collected
    CompletableFuture<?> future = dynamicFilter.isBlocked();
    remoteTask.start();
    future.get();
    assertEquals(dynamicFilter.getCurrentPredicate(), TupleDomain.withColumnDomains(ImmutableMap.of(handle1, Domain.singleValue(BIGINT, 1L))));
    assertEquals(testingTaskResource.getDynamicFiltersFetchCounter(), 1);
    // make sure dynamic filters are not collected for every status update
    assertEventually(new Duration(15, SECONDS), () -> assertGreaterThanOrEqual(testingTaskResource.getStatusFetchCounter(), 3L));
    assertEquals(testingTaskResource.getDynamicFiltersFetchCounter(), 1L, testingTaskResource.getDynamicFiltersFetchRequests().toString());
    future = dynamicFilter.isBlocked();
    testingTaskResource.setDynamicFilterDomains(new VersionedDynamicFilterDomains(2L, ImmutableMap.of(filterId2, Domain.singleValue(BIGINT, 2L))));
    future.get();
    assertEquals(dynamicFilter.getCurrentPredicate(), TupleDomain.withColumnDomains(ImmutableMap.of(handle1, Domain.singleValue(BIGINT, 1L), handle2, Domain.singleValue(BIGINT, 2L))));
    assertEquals(testingTaskResource.getDynamicFiltersFetchCounter(), 2L, testingTaskResource.getDynamicFiltersFetchRequests().toString());
    assertGreaterThanOrEqual(testingTaskResource.getStatusFetchCounter(), 4L);
    httpRemoteTaskFactory.stop();
    dynamicFilterService.stop();
}