Examples with PagesSerdeFactory io.hetu.core.transport.execution.buffer.PagesSerdeFactory used on opensource projects

Example 1 with PagesSerdeFactory

use of io.hetu.core.transport.execution.buffer.PagesSerdeFactory in project hetu-core by openlookeng.

the class TestBinaryFileSpiller method setUp.

@BeforeMethod
public void setUp() throws IOException {
    Metadata metadata = createTestMetadataManager();
    FileSystemClientManager fileSystemClientManager = mock(FileSystemClientManager.class);
    when(fileSystemClientManager.getFileSystemClient(any(Path.class))).thenReturn(new HetuLocalFileSystemClient(new LocalConfig(new Properties()), Paths.get(spillPath.getCanonicalPath())));
    spillerStats = new SpillerStats();
    FeaturesConfig featuresConfig = new FeaturesConfig();
    try {
        featuresConfig.setSpillerSpillPaths(spillPath.getCanonicalPath());
    } catch (IOException e) {
        System.out.println(e.getStackTrace());
    }
    featuresConfig.setSpillMaxUsedSpaceThreshold(1.0);
    NodeSpillConfig nodeSpillConfig = new NodeSpillConfig();
    singleStreamSpillerFactory = new FileSingleStreamSpillerFactory(metadata, spillerStats, featuresConfig, nodeSpillConfig, fileSystemClientManager);
    factory = new GenericSpillerFactory(singleStreamSpillerFactory);
    PagesSerdeFactory pagesSerdeFactory = new PagesSerdeFactory(metadata.getFunctionAndTypeManager().getBlockEncodingSerde(), nodeSpillConfig.isSpillCompressionEnabled());
    pagesSerde = pagesSerdeFactory.createPagesSerde();
    memoryContext = newSimpleAggregatedMemoryContext();
}

Example 2 with PagesSerdeFactory

use of io.hetu.core.transport.execution.buffer.PagesSerdeFactory in project hetu-core by openlookeng.

the class SqlTaskExecutionFactory method create.

public SqlTaskExecution create(String taskInstanceId, Session session, QueryContext queryContext, TaskStateMachine taskStateMachine, OutputBuffer outputBuffer, PlanFragment fragment, List<TaskSource> sources, OptionalInt totalPartitions, Optional<PlanNodeId> consumer, Map<String, CommonTableExecutionContext> cteCtx) {
    TaskContext taskContext = queryContext.addTaskContext(taskInstanceId, taskStateMachine, session, perOperatorCpuTimerEnabled, cpuTimerEnabled, totalPartitions, consumer, new PagesSerdeFactory(metadata.getFunctionAndTypeManager().getBlockEncodingSerde(), isExchangeCompressionEnabled(session)));
    LocalExecutionPlan localExecutionPlan;
    try (SetThreadName ignored = new SetThreadName("Task-%s", taskStateMachine.getTaskId())) {
        try {
            localExecutionPlan = planner.plan(taskContext, fragment.getRoot(), TypeProvider.copyOf(fragment.getSymbols()), fragment.getPartitioningScheme(), fragment.getStageExecutionDescriptor(), fragment.getPartitionedSources(), outputBuffer, fragment.getFeederCTEId(), fragment.getFeederCTEParentId(), cteCtx);
        } catch (Throwable e) {
            // planning failed
            taskStateMachine.failed(e);
            throwIfUnchecked(e);
            throw new RuntimeException(e);
        }
    }
    return createSqlTaskExecution(taskStateMachine, taskContext, outputBuffer, sources, localExecutionPlan, taskExecutor, taskNotificationExecutor, splitMonitor);
}

Example 3 with PagesSerdeFactory

use of io.hetu.core.transport.execution.buffer.PagesSerdeFactory in project hetu-core by openlookeng.

the class MemoryConnectorFactory method create.

@Override
public Connector create(String catalogName, Map<String, String> requiredConfig, ConnectorContext context) {
    requireNonNull(requiredConfig, "requiredConfig is null");
    if (context.getHetuMetastore() == null) {
        throw new IllegalStateException("HetuMetastore must be configured to use the Memory Connector. Please refer to HetuMetastore docs for details.");
    }
    try {
        // A plugin is not required to use Guice; it is just very convenient
        Bootstrap app = new Bootstrap(binder -> {
            binder.bind(PageSorter.class).toInstance(context.getPageSorter());
            binder.bind(FunctionMetadataManager.class).toInstance(context.getFunctionMetadataManager());
            binder.bind(StandardFunctionResolution.class).toInstance(context.getStandardFunctionResolution());
            binder.bind(TypeManager.class).toInstance(context.getTypeManager());
            binder.bind(RowExpressionService.class).toInstance(context.getRowExpressionService());
            binder.bind(DeterminismEvaluator.class).toInstance(context.getRowExpressionService().getDeterminismEvaluator());
            binder.bind(HetuMetastore.class).toInstance(context.getHetuMetastore());
        }, new MBeanModule(), new MBeanServerModule(), new JsonModule(), new MemoryModule(context.getTypeManager(), context.getNodeManager(), new PagesSerdeFactory(context.getBlockEncodingSerde(), false).createPagesSerde()));
        Injector injector = app.strictConfig().doNotInitializeLogging().setRequiredConfigurationProperties(requiredConfig).initialize();
        synchronized (MemoryThreadManager.class) {
            if (!MemoryThreadManager.isSharedThreadPoolInitilized()) {
                MemoryThreadManager.initSharedThreadPool(injector.getInstance(MemoryConfig.class).getThreadPoolSize());
            }
        }
        MemoryConnector memoryConnector = injector.getInstance(MemoryConnector.class);
        memoryConnector.scheduleRefreshJob();
        return memoryConnector;
    } catch (Exception e) {
        throwIfUnchecked(e);
        throw new RuntimeException(e);
    }
}