Examples with DataSize io.airlift.units.DataSize used on opensource projects

Example 86 with DataSize

use of io.airlift.units.DataSize in project presto by prestodb.

the class StatusPrinter method printStageTree.

private void printStageTree(StageStats stage, String indent, AtomicInteger stageNumberCounter) {
    Duration elapsedTime = nanosSince(start);
    // STAGE  S    ROWS  ROWS/s  BYTES  BYTES/s  QUEUED    RUN   DONE
    // 0......Q     26M   9077M  9993G    9077M   9077M  9077M  9077M
    //   2....R     17K    627M   673M     627M    627M   627M   627M
    //     3..C     999    627M   673M     627M    627M   627M   627M
    //   4....R     26M    627M   673T     627M    627M   627M   627M
    //     5..F     29T    627M   673M     627M    627M   627M   627M
    String id = String.valueOf(stageNumberCounter.getAndIncrement());
    String name = indent + id;
    name += Strings.repeat(".", max(0, 10 - name.length()));
    String bytesPerSecond;
    String rowsPerSecond;
    if (stage.isDone()) {
        bytesPerSecond = formatDataRate(new DataSize(0, BYTE), new Duration(0, SECONDS), false);
        rowsPerSecond = formatCountRate(0, new Duration(0, SECONDS), false);
    } else {
        bytesPerSecond = formatDataRate(bytes(stage.getProcessedBytes()), elapsedTime, false);
        rowsPerSecond = formatCountRate(stage.getProcessedRows(), elapsedTime, false);
    }
    String stageSummary = String.format("%10s%1s  %5s  %6s  %5s  %7s  %6s  %5s  %5s", name, stageStateCharacter(stage.getState()), formatCount(stage.getProcessedRows()), rowsPerSecond, formatDataSize(bytes(stage.getProcessedBytes()), false), bytesPerSecond, stage.getQueuedSplits(), stage.getRunningSplits(), stage.getCompletedSplits());
    reprintLine(stageSummary);
    for (StageStats subStage : stage.getSubStages()) {
        printStageTree(subStage, indent + "  ", stageNumberCounter);
    }
}

Example 87 with DataSize

use of io.airlift.units.DataSize in project presto by prestodb.

the class TestMemoryPagesStore method setUp.

@BeforeMethod
public void setUp() {
    pagesStore = new MemoryPagesStore(new MemoryConfig().setMaxDataPerNode(new DataSize(1, DataSize.Unit.MEGABYTE)));
    pageSinkProvider = new MemoryPageSinkProvider(pagesStore);
}

Example 88 with DataSize

use of io.airlift.units.DataSize in project presto by prestodb.

the class HiveFileFormatBenchmark method createTpchDataSet.

private static <E extends TpchEntity> TestData createTpchDataSet(FileFormat format, TpchTable<E> tpchTable, List<TpchColumn<E>> columns) {
    List<String> columnNames = columns.stream().map(TpchColumn::getColumnName).collect(toList());
    List<Type> columnTypes = columns.stream().map(HiveFileFormatBenchmark::getColumnType).map(type -> format.supportsDate() || !DATE.equals(type) ? type : createUnboundedVarcharType()).collect(toList());
    PageBuilder pageBuilder = new PageBuilder(columnTypes);
    ImmutableList.Builder<Page> pages = ImmutableList.builder();
    long dataSize = 0;
    for (E row : tpchTable.createGenerator(10, 1, 1)) {
        pageBuilder.declarePosition();
        for (int i = 0; i < columns.size(); i++) {
            TpchColumn<E> column = columns.get(i);
            BlockBuilder blockBuilder = pageBuilder.getBlockBuilder(i);
            switch(column.getType().getBase()) {
                case IDENTIFIER:
                    BIGINT.writeLong(blockBuilder, column.getIdentifier(row));
                    break;
                case INTEGER:
                    INTEGER.writeLong(blockBuilder, column.getInteger(row));
                    break;
                case DATE:
                    if (format.supportsDate()) {
                        DATE.writeLong(blockBuilder, column.getDate(row));
                    } else {
                        createUnboundedVarcharType().writeString(blockBuilder, column.getString(row));
                    }
                    break;
                case DOUBLE:
                    DOUBLE.writeDouble(blockBuilder, column.getDouble(row));
                    break;
                case VARCHAR:
                    createUnboundedVarcharType().writeSlice(blockBuilder, Slices.utf8Slice(column.getString(row)));
                    break;
                default:
                    throw new IllegalArgumentException("Unsupported type " + column.getType());
            }
        }
        if (pageBuilder.isFull()) {
            Page page = pageBuilder.build();
            pages.add(page);
            pageBuilder.reset();
            dataSize += page.getSizeInBytes();
            if (dataSize >= MIN_DATA_SIZE) {
                break;
            }
        }
    }
    return new TestData(columnNames, columnTypes, pages.build());
}

Example 89 with DataSize

use of io.airlift.units.DataSize in project presto by prestodb.

the class RcFileTester method writeRcFileColumnOld.

private static DataSize writeRcFileColumnOld(File outputFile, Format format, Compression compression, Type type, Iterator<?> values) throws Exception {
    ObjectInspector columnObjectInspector = getJavaObjectInspector(type);
    RecordWriter recordWriter = createRcFileWriterOld(outputFile, compression, columnObjectInspector);
    SettableStructObjectInspector objectInspector = createSettableStructObjectInspector("test", columnObjectInspector);
    Object row = objectInspector.create();
    List<StructField> fields = ImmutableList.copyOf(objectInspector.getAllStructFieldRefs());
    @SuppressWarnings("deprecation") Serializer serializer = format.createSerializer();
    Properties tableProperties = new Properties();
    tableProperties.setProperty("columns", "test");
    tableProperties.setProperty("columns.types", objectInspector.getTypeName());
    serializer.initialize(new JobConf(false), tableProperties);
    while (values.hasNext()) {
        Object value = values.next();
        value = preprocessWriteValueOld(type, value);
        objectInspector.setStructFieldData(row, fields.get(0), value);
        Writable record = serializer.serialize(row, objectInspector);
        recordWriter.write(record);
    }
    recordWriter.close(false);
    return new DataSize(outputFile.length(), BYTE).convertToMostSuccinctDataSize();
}

Example 90 with DataSize

use of io.airlift.units.DataSize in project presto by prestodb.

the class TestingTaskContext method createTaskContext.

public static TaskContext createTaskContext(Executor executor, Session session, DataSize maxMemory) {
    MemoryPool memoryPool = new MemoryPool(new MemoryPoolId("test"), new DataSize(1, GIGABYTE));
    MemoryPool systemMemoryPool = new MemoryPool(new MemoryPoolId("testSystem"), new DataSize(1, GIGABYTE));
    QueryContext queryContext = new QueryContext(new QueryId("test_query"), maxMemory, memoryPool, systemMemoryPool, executor);
    return createTaskContext(queryContext, executor, session);
}