Examples with Slice io.airlift.slice.Slice used on opensource projects

Example 1 with Slice

use of io.airlift.slice.Slice in project carbondata by apache.

the class CarbondataSplitManager method parseFilterExpression.

/**
   * Convert presto-TupleDomain predication into Carbon scan express condition
   * @param originalConstraint  presto-TupleDomain
   * @param carbonTable
   * @return
   */
public Expression parseFilterExpression(TupleDomain<ColumnHandle> originalConstraint, CarbonTable carbonTable) {
    ImmutableList.Builder<Expression> filters = ImmutableList.builder();
    Domain domain = null;
    for (ColumnHandle c : originalConstraint.getDomains().get().keySet()) {
        CarbondataColumnHandle cdch = (CarbondataColumnHandle) c;
        Type type = cdch.getColumnType();
        List<CarbonColumn> ccols = carbonTable.getCreateOrderColumn(carbonTable.getFactTableName());
        Optional<CarbonColumn> target = ccols.stream().filter(a -> a.getColName().equals(cdch.getColumnName())).findFirst();
        if (target.get() == null)
            return null;
        DataType coltype = target.get().getDataType();
        ColumnExpression colExpression = new ColumnExpression(cdch.getColumnName(), target.get().getDataType());
        //colExpression.setColIndex(cs.getSchemaOrdinal());
        colExpression.setDimension(target.get().isDimension());
        colExpression.setDimension(carbonTable.getDimensionByName(carbonTable.getFactTableName(), cdch.getColumnName()));
        colExpression.setCarbonColumn(target.get());
        domain = originalConstraint.getDomains().get().get(c);
        checkArgument(domain.getType().isOrderable(), "Domain type must be orderable");
        if (domain.getValues().isNone()) {
        }
        if (domain.getValues().isAll()) {
        }
        List<Object> singleValues = new ArrayList<>();
        List<Expression> rangeFilter = new ArrayList<>();
        for (Range range : domain.getValues().getRanges().getOrderedRanges()) {
            // Already checked
            checkState(!range.isAll());
            if (range.isSingleValue()) {
                singleValues.add(range.getLow().getValue());
            } else {
                List<String> rangeConjuncts = new ArrayList<>();
                if (!range.getLow().isLowerUnbounded()) {
                    Object value = ConvertDataByType(range.getLow().getValue(), type);
                    switch(range.getLow().getBound()) {
                        case ABOVE:
                            if (type == TimestampType.TIMESTAMP) {
                            //todo not now
                            } else {
                                GreaterThanExpression greater = new GreaterThanExpression(colExpression, new LiteralExpression(value, coltype));
                                rangeFilter.add(greater);
                            }
                            break;
                        case EXACTLY:
                            GreaterThanEqualToExpression greater = new GreaterThanEqualToExpression(colExpression, new LiteralExpression(value, coltype));
                            rangeFilter.add(greater);
                            break;
                        case BELOW:
                            throw new IllegalArgumentException("Low marker should never use BELOW bound");
                        default:
                            throw new AssertionError("Unhandled bound: " + range.getLow().getBound());
                    }
                }
                if (!range.getHigh().isUpperUnbounded()) {
                    Object value = ConvertDataByType(range.getHigh().getValue(), type);
                    switch(range.getHigh().getBound()) {
                        case ABOVE:
                            throw new IllegalArgumentException("High marker should never use ABOVE bound");
                        case EXACTLY:
                            LessThanEqualToExpression less = new LessThanEqualToExpression(colExpression, new LiteralExpression(value, coltype));
                            rangeFilter.add(less);
                            break;
                        case BELOW:
                            LessThanExpression less2 = new LessThanExpression(colExpression, new LiteralExpression(value, coltype));
                            rangeFilter.add(less2);
                            break;
                        default:
                            throw new AssertionError("Unhandled bound: " + range.getHigh().getBound());
                    }
                }
            }
        }
        if (singleValues.size() == 1) {
            Expression ex = null;
            if (coltype.equals(DataType.STRING)) {
                ex = new EqualToExpression(colExpression, new LiteralExpression(((Slice) singleValues.get(0)).toStringUtf8(), coltype));
            } else
                ex = new EqualToExpression(colExpression, new LiteralExpression(singleValues.get(0), coltype));
            filters.add(ex);
        } else if (singleValues.size() > 1) {
            ListExpression candidates = null;
            List<Expression> exs = singleValues.stream().map((a) -> {
                return new LiteralExpression(ConvertDataByType(a, type), coltype);
            }).collect(Collectors.toList());
            candidates = new ListExpression(exs);
            if (candidates != null)
                filters.add(new InExpression(colExpression, candidates));
        } else if (rangeFilter.size() > 0) {
            if (rangeFilter.size() > 1) {
                Expression finalFilters = new OrExpression(rangeFilter.get(0), rangeFilter.get(1));
                if (rangeFilter.size() > 2) {
                    for (int i = 2; i < rangeFilter.size(); i++) {
                        filters.add(new AndExpression(finalFilters, rangeFilter.get(i)));
                    }
                }
            } else if (//only have one value
            rangeFilter.size() == 1)
                filters.add(rangeFilter.get(0));
        }
    }
    Expression finalFilters;
    List<Expression> tmp = filters.build();
    if (tmp.size() > 1) {
        finalFilters = new AndExpression(tmp.get(0), tmp.get(1));
        if (tmp.size() > 2) {
            for (int i = 2; i < tmp.size(); i++) {
                finalFilters = new AndExpression(finalFilters, tmp.get(i));
            }
        }
    } else if (tmp.size() == 1)
        finalFilters = tmp.get(0);
    else
        //no filter
        return null;
    return finalFilters;
}

Example 2 with Slice

use of io.airlift.slice.Slice in project presto by prestodb.

the class TupleDomainParquetPredicate method getDomain.

@VisibleForTesting
public static Domain getDomain(Type type, long rowCount, Statistics<?> statistics) {
    if (statistics == null || statistics.isEmpty()) {
        return Domain.all(type);
    }
    if (statistics.getNumNulls() == rowCount) {
        return Domain.onlyNull(type);
    }
    boolean hasNullValue = statistics.getNumNulls() != 0L;
    // ignore corrupted statistics
    if (statistics.genericGetMin() == null || statistics.genericGetMax() == null) {
        return Domain.create(ValueSet.all(type), hasNullValue);
    }
    if (type.equals(BOOLEAN) && statistics instanceof BooleanStatistics) {
        BooleanStatistics booleanStatistics = (BooleanStatistics) statistics;
        boolean hasTrueValues = !(booleanStatistics.getMax() == false && booleanStatistics.getMin() == false);
        boolean hasFalseValues = !(booleanStatistics.getMax() == true && booleanStatistics.getMin() == true);
        if (hasTrueValues && hasFalseValues) {
            return Domain.all(type);
        }
        if (hasTrueValues) {
            return Domain.create(ValueSet.of(type, true), hasNullValue);
        }
        if (hasFalseValues) {
            return Domain.create(ValueSet.of(type, false), hasNullValue);
        }
    } else if ((type.equals(BIGINT) || type.equals(TINYINT) || type.equals(SMALLINT) || type.equals(INTEGER)) && (statistics instanceof LongStatistics || statistics instanceof IntStatistics)) {
        ParquetIntegerStatistics parquetIntegerStatistics;
        if (statistics instanceof LongStatistics) {
            LongStatistics longStatistics = (LongStatistics) statistics;
            // ignore corrupted statistics
            if (longStatistics.genericGetMin() > longStatistics.genericGetMax()) {
                return Domain.create(ValueSet.all(type), hasNullValue);
            }
            parquetIntegerStatistics = new ParquetIntegerStatistics(longStatistics.genericGetMin(), longStatistics.genericGetMax());
        } else {
            IntStatistics intStatistics = (IntStatistics) statistics;
            // ignore corrupted statistics
            if (intStatistics.genericGetMin() > intStatistics.genericGetMax()) {
                return Domain.create(ValueSet.all(type), hasNullValue);
            }
            parquetIntegerStatistics = new ParquetIntegerStatistics((long) intStatistics.getMin(), (long) intStatistics.getMax());
        }
        if (isStatisticsOverflow(type, parquetIntegerStatistics)) {
            return Domain.create(ValueSet.all(type), hasNullValue);
        }
        return createDomain(type, hasNullValue, parquetIntegerStatistics);
    } else if (type.equals(REAL) && statistics instanceof FloatStatistics) {
        FloatStatistics floatStatistics = (FloatStatistics) statistics;
        // ignore corrupted statistics
        if (floatStatistics.genericGetMin() > floatStatistics.genericGetMax()) {
            return Domain.create(ValueSet.all(type), hasNullValue);
        }
        ParquetIntegerStatistics parquetStatistics = new ParquetIntegerStatistics((long) floatToRawIntBits(floatStatistics.getMin()), (long) floatToRawIntBits(floatStatistics.getMax()));
        return createDomain(type, hasNullValue, parquetStatistics);
    } else if (type.equals(DOUBLE) && statistics instanceof DoubleStatistics) {
        DoubleStatistics doubleStatistics = (DoubleStatistics) statistics;
        // ignore corrupted statistics
        if (doubleStatistics.genericGetMin() > doubleStatistics.genericGetMax()) {
            return Domain.create(ValueSet.all(type), hasNullValue);
        }
        ParquetDoubleStatistics parquetDoubleStatistics = new ParquetDoubleStatistics(doubleStatistics.genericGetMin(), doubleStatistics.genericGetMax());
        return createDomain(type, hasNullValue, parquetDoubleStatistics);
    } else if (isVarcharType(type) && statistics instanceof BinaryStatistics) {
        BinaryStatistics binaryStatistics = (BinaryStatistics) statistics;
        Slice minSlice = Slices.wrappedBuffer(binaryStatistics.getMin().getBytes());
        Slice maxSlice = Slices.wrappedBuffer(binaryStatistics.getMax().getBytes());
        // ignore corrupted statistics
        if (minSlice.compareTo(maxSlice) > 0) {
            return Domain.create(ValueSet.all(type), hasNullValue);
        }
        ParquetStringStatistics parquetStringStatistics = new ParquetStringStatistics(minSlice, maxSlice);
        return createDomain(type, hasNullValue, parquetStringStatistics);
    }
    return Domain.create(ValueSet.all(type), hasNullValue);
}

Example 3 with Slice

use of io.airlift.slice.Slice in project presto by prestodb.

the class DecimalStream method nextLongDecimalVector.

public void nextLongDecimalVector(int items, BlockBuilder builder, DecimalType targetType, long[] sourceScale) throws IOException {
    Slice decimal = UnscaledDecimal128Arithmetic.unscaledDecimal();
    Slice rescaledDecimal = UnscaledDecimal128Arithmetic.unscaledDecimal();
    for (int i = 0; i < items; i++) {
        nextLongDecimal(decimal);
        rescale(decimal, (int) (targetType.getScale() - sourceScale[i]), rescaledDecimal);
        targetType.writeSlice(builder, rescaledDecimal);
    }
}

Example 4 with Slice

use of io.airlift.slice.Slice in project presto by prestodb.

the class DecimalStream method nextLongDecimalVector.

public void nextLongDecimalVector(int items, BlockBuilder builder, DecimalType targetType, long[] sourceScale, boolean[] isNull) throws IOException {
    Slice decimal = UnscaledDecimal128Arithmetic.unscaledDecimal();
    Slice rescaledDecimal = UnscaledDecimal128Arithmetic.unscaledDecimal();
    for (int i = 0; i < items; i++) {
        if (!isNull[i]) {
            nextLongDecimal(decimal);
            rescale(decimal, (int) (targetType.getScale() - sourceScale[i]), rescaledDecimal);
            targetType.writeSlice(builder, rescaledDecimal);
        } else {
            builder.appendNull();
        }
    }
}

Example 5 with Slice

use of io.airlift.slice.Slice in project presto by prestodb.

the class AbstractTestHiveClient method doTestMismatchSchemaTable.

protected void doTestMismatchSchemaTable(SchemaTableName schemaTableName, HiveStorageFormat storageFormat, List<ColumnMetadata> tableBefore, MaterializedResult dataBefore, List<ColumnMetadata> tableAfter, MaterializedResult dataAfter) throws Exception {
    String schemaName = schemaTableName.getSchemaName();
    String tableName = schemaTableName.getTableName();
    doCreateEmptyTable(schemaTableName, storageFormat, tableBefore);
    // insert the data
    try (Transaction transaction = newTransaction()) {
        ConnectorSession session = newSession();
        ConnectorMetadata metadata = transaction.getMetadata();
        ConnectorTableHandle tableHandle = getTableHandle(metadata, schemaTableName);
        ConnectorInsertTableHandle insertTableHandle = metadata.beginInsert(session, tableHandle);
        ConnectorPageSink sink = pageSinkProvider.createPageSink(transaction.getTransactionHandle(), session, insertTableHandle);
        sink.appendPage(dataBefore.toPage());
        Collection<Slice> fragments = getFutureValue(sink.finish());
        metadata.finishInsert(session, insertTableHandle, fragments);
        transaction.commit();
    }
    // load the table and verify the data
    try (Transaction transaction = newTransaction()) {
        ConnectorSession session = newSession();
        ConnectorMetadata metadata = transaction.getMetadata();
        ConnectorTableHandle tableHandle = getTableHandle(metadata, schemaTableName);
        List<ColumnHandle> columnHandles = metadata.getColumnHandles(session, tableHandle).values().stream().filter(columnHandle -> !((HiveColumnHandle) columnHandle).isHidden()).collect(toList());
        MaterializedResult result = readTable(transaction, tableHandle, columnHandles, session, TupleDomain.all(), OptionalInt.empty(), Optional.empty());
        assertEqualsIgnoreOrder(result.getMaterializedRows(), dataBefore.getMaterializedRows());
        transaction.commit();
    }
    // alter the table schema
    try (Transaction transaction = newTransaction()) {
        ConnectorSession session = newSession();
        PrincipalPrivileges principalPrivileges = new PrincipalPrivileges(ImmutableMultimap.<String, HivePrivilegeInfo>builder().put(session.getUser(), new HivePrivilegeInfo(HivePrivilege.SELECT, true)).put(session.getUser(), new HivePrivilegeInfo(HivePrivilege.INSERT, true)).put(session.getUser(), new HivePrivilegeInfo(HivePrivilege.UPDATE, true)).put(session.getUser(), new HivePrivilegeInfo(HivePrivilege.DELETE, true)).build(), ImmutableMultimap.of());
        Table oldTable = transaction.getMetastore(schemaName).getTable(schemaName, tableName).get();
        HiveTypeTranslator hiveTypeTranslator = new HiveTypeTranslator();
        List<Column> dataColumns = tableAfter.stream().filter(columnMetadata -> !columnMetadata.getName().equals("ds")).map(columnMetadata -> new Column(columnMetadata.getName(), toHiveType(hiveTypeTranslator, columnMetadata.getType()), Optional.empty())).collect(toList());
        Table.Builder newTable = Table.builder(oldTable).setDataColumns(dataColumns);
        transaction.getMetastore(schemaName).replaceView(schemaName, tableName, newTable.build(), principalPrivileges);
        transaction.commit();
    }
    // load the altered table and verify the data
    try (Transaction transaction = newTransaction()) {
        ConnectorSession session = newSession();
        ConnectorMetadata metadata = transaction.getMetadata();
        ConnectorTableHandle tableHandle = getTableHandle(metadata, schemaTableName);
        List<ColumnHandle> columnHandles = metadata.getColumnHandles(session, tableHandle).values().stream().filter(columnHandle -> !((HiveColumnHandle) columnHandle).isHidden()).collect(toList());
        MaterializedResult result = readTable(transaction, tableHandle, columnHandles, session, TupleDomain.all(), OptionalInt.empty(), Optional.empty());
        assertEqualsIgnoreOrder(result.getMaterializedRows(), dataAfter.getMaterializedRows());
        transaction.commit();
    }
    // insertions to the partitions with type mismatches should fail
    try (Transaction transaction = newTransaction()) {
        ConnectorSession session = newSession();
        ConnectorMetadata metadata = transaction.getMetadata();
        ConnectorTableHandle tableHandle = getTableHandle(metadata, schemaTableName);
        ConnectorInsertTableHandle insertTableHandle = metadata.beginInsert(session, tableHandle);
        ConnectorPageSink sink = pageSinkProvider.createPageSink(transaction.getTransactionHandle(), session, insertTableHandle);
        sink.appendPage(dataAfter.toPage());
        Collection<Slice> fragments = getFutureValue(sink.finish());
        metadata.finishInsert(session, insertTableHandle, fragments);
        transaction.commit();
        fail("expected exception");
    } catch (PrestoException e) {
        // expected
        assertEquals(e.getErrorCode(), HIVE_PARTITION_SCHEMA_MISMATCH.toErrorCode());
    }
}