Examples with Spiller io.prestosql.spiller.Spiller used on opensource projects

Example 1 with Spiller

use of io.prestosql.spiller.Spiller in project hetu-core by openlookeng.

the class TableScanOperator method setPage.

private void setPage(Page page) {
    synchronized (reuseExchangeTableScanMappingIdUtilsMap) {
        if (!spillEnabled) {
            // spilling is not enabled so keep adding pages to cache in-memory
            List<Page> pageCachesList = reuseExchangeTableScanMappingIdState.getPageCaches();
            pageCachesList.add(page);
            reuseExchangeTableScanMappingIdState.setPageCaches(pageCachesList);
        } else {
            if (totalPageSize(reuseExchangeTableScanMappingIdState.getPageCaches()) < (spillThreshold / 2)) {
                // if pageCaches hasn't reached spillThreshold/2, keep adding pages to it.
                List<Page> pageCachesList = reuseExchangeTableScanMappingIdState.getPageCaches();
                pageCachesList.add(page);
                reuseExchangeTableScanMappingIdState.setPageCaches(pageCachesList);
            } else {
                // no more space available in memory to store pages. pages will be spilled now
                List<Page> pageSpilledList = reuseExchangeTableScanMappingIdState.getPagesToSpill();
                pageSpilledList.add(page);
                reuseExchangeTableScanMappingIdState.setPagesToSpill(pageSpilledList);
                if (totalPageSize(pageSpilledList) >= (spillThreshold / 2)) {
                    if (!reuseExchangeTableScanMappingIdState.getSpiller().isPresent()) {
                        Optional<Spiller> spillObject = Optional.of(spillerFactory.get().create(types, operatorContext.getSpillContext(), operatorContext.newAggregateSystemMemoryContext()));
                        reuseExchangeTableScanMappingIdState.setSpiller(spillObject);
                    }
                    spillInProgress = reuseExchangeTableScanMappingIdState.getSpiller().get().spill(pageSpilledList.iterator());
                    LOG.debug("spilling to disk initiated by reuse exchange");
                    try {
                        // blocking call to ensure spilling completes before we move forward
                        spillInProgress.get();
                    } catch (InterruptedException | ExecutionException e) {
                        cleanupInErrorCase();
                        throw new PrestoException(GENERIC_INTERNAL_ERROR, e.getMessage(), e);
                    }
                    reuseExchangeTableScanMappingIdState.setPagesWritten(reuseExchangeTableScanMappingIdState.getPagesWrittenCount() + pageSpilledList.size());
                    // clear the memory pressure once the data is spilled to disk
                    reuseExchangeTableScanMappingIdState.clearPagesToSpill();
                }
            }
        }
    }
}

Example 2 with Spiller

use of io.prestosql.spiller.Spiller in project hetu-core by openlookeng.

the class DummySpillerFactory method create.

@Override
public Spiller create(List<Type> types, SpillContext spillContext, AggregatedMemoryContext memoryContext) {
    return new Spiller() {

        @RestorableConfig(unsupported = true)
        private final RestorableConfig restorableConfig = null;

        private final List<Iterable<Page>> spills = new ArrayList<>();

        private final List<AtomicBoolean> spillCommitted = new ArrayList<>();

        @Override
        public ListenableFuture<?> spill(Iterator<Page> pageIterator) {
            spillsCount++;
            spills.add(ImmutableList.copyOf(pageIterator));
            spillCommitted.add(new AtomicBoolean(true));
            return immediateFuture(null);
        }

        @Override
        public Pair<ListenableFuture<?>, Runnable> spillUnCommit(Iterator<Page> pageIterator) {
            spillsCount++;
            spills.add(ImmutableList.copyOf(pageIterator));
            AtomicBoolean isCommitted = new AtomicBoolean(false);
            spillCommitted.add(isCommitted);
            return ImmutablePair.of(immediateFuture(null), () -> isCommitted.set(true));
        }

        @Override
        public List<Iterator<Page>> getSpills() {
            return spills.stream().map(Iterable::iterator).collect(toImmutableList());
        }

        @Override
        public void close() {
            spills.clear();
        }

        /**
         * Capture this object's internal state, so it can be used later to restore to the same state.
         *
         * @param serdeProvider
         * @return An object representing internal state of the current object
         */
        @Override
        public Object capture(BlockEncodingSerdeProvider serdeProvider) {
            DummySpillerState myState = new DummySpillerState();
            for (int i = 0; i < spills.size(); i++) {
                if (spillCommitted.get(i).get()) {
                    List<Page> pages = new ArrayList<>();
                    spills.get(i).forEach(pg -> pages.add(pg));
                    myState.spills.add(pages);
                }
            }
            return myState;
        }

        /**
         * Restore this object's internal state according to the snapshot
         *
         * @param state         an object that represents this object's snapshot state
         * @param serdeProvider
         */
        @Override
        public void restore(Object state, BlockEncodingSerdeProvider serdeProvider) {
            DummySpillerState myState = (DummySpillerState) state;
            this.spills.clear();
            for (List<Page> s : myState.spills) {
                this.spills.add(s);
                this.spillCommitted.add(new AtomicBoolean(true));
            }
        }

        class DummySpillerState implements Serializable {

            List<List<Page>> spills = new ArrayList<>();
        }
    };
}

Example 3 with Spiller

use of io.prestosql.spiller.Spiller in project hetu-core by openlookeng.

the class BenchmarkBinaryFileSpiller method writeReadSpill.

@Benchmark
public void writeReadSpill(BenchmarkData data) throws ExecutionException, InterruptedException {
    try (Spiller spiller = data.createSpiller()) {
        spiller.spill(data.getPages().iterator()).get();
        List<Iterator<Page>> spills = spiller.getSpills();
        for (Iterator<Page> spill : spills) {
            while (spill.hasNext()) {
                Page next = spill.next();
                next.getPositionCount();
            }
        }
    }
}

Example 4 with Spiller

use of io.prestosql.spiller.Spiller in project hetu-core by openlookeng.

the class WorkProcessorSourceOperatorAdapter method setPage.

private void setPage(Page page) {
    synchronized (reuseExchangeTableScanMappingIdState) {
        if (!spillEnabled) {
            // spilling is not enabled so keep adding pages to cache in-memory
            List<Page> pageCachesList = reuseExchangeTableScanMappingIdState.getPageCaches();
            pageCachesList.add(page);
            reuseExchangeTableScanMappingIdState.setPageCaches(pageCachesList);
        } else {
            if (totalPageSize(reuseExchangeTableScanMappingIdState.getPageCaches()) < (spillThreshold / 2)) {
                // if pageCaches hasn't reached spillThreshold/2, keep adding pages to it.
                List<Page> pageCachesList = reuseExchangeTableScanMappingIdState.getPageCaches();
                pageCachesList.add(page);
                reuseExchangeTableScanMappingIdState.setPageCaches(pageCachesList);
            } else {
                // no more space available in memory to store pages. pages will be spilled now
                List<Page> pageSpilledList = reuseExchangeTableScanMappingIdState.getPagesToSpill();
                pageSpilledList.add(page);
                reuseExchangeTableScanMappingIdState.setPagesToSpill(pageSpilledList);
                if (totalPageSize(pageSpilledList) >= (spillThreshold / 2)) {
                    if (!reuseExchangeTableScanMappingIdState.getSpiller().isPresent()) {
                        Optional<Spiller> spillObject = Optional.of(spillerFactory.get().create(projectionTypes, operatorContext.getSpillContext(), operatorContext.newAggregateSystemMemoryContext()));
                        reuseExchangeTableScanMappingIdState.setSpiller(spillObject);
                    }
                    spillInProgress = reuseExchangeTableScanMappingIdState.getSpiller().get().spill(pageSpilledList.iterator());
                    LOG.debug("spilling to disk initiated by reuse exchange");
                    try {
                        // blocking call to ensure spilling completes before we move forward
                        spillInProgress.get();
                    } catch (InterruptedException | ExecutionException e) {
                        cleanupInErrorCase();
                        throw new PrestoException(GENERIC_INTERNAL_ERROR, e.getMessage(), e);
                    }
                    reuseExchangeTableScanMappingIdState.setPagesWritten(reuseExchangeTableScanMappingIdState.getPagesWrittenCount() + pageSpilledList.size());
                    // clear the memory pressure once the data is spilled to disk
                    reuseExchangeTableScanMappingIdState.clearPagesToSpill();
                }
            }
        }
    }
}