Examples with Expression org.apache.iceberg.expressions.Expression used on opensource projects

Example 16 with Expression

use of org.apache.iceberg.expressions.Expression in project presto by prestodb.

the class IcebergUtil method getTableScan.

public static TableScan getTableScan(TupleDomain<IcebergColumnHandle> predicates, Optional<Long> snapshotId, Table icebergTable) {
    Expression expression = ExpressionConverter.toIcebergExpression(predicates);
    TableScan tableScan = icebergTable.newScan().filter(expression);
    return snapshotId.map(id -> isSnapshot(icebergTable, id) ? tableScan.useSnapshot(id) : tableScan.asOfTime(id)).orElse(tableScan);
}

Example 17 with Expression

use of org.apache.iceberg.expressions.Expression in project hive by apache.

the class HiveIcebergInputFormat method getSplits.

@Override
public InputSplit[] getSplits(JobConf job, int numSplits) throws IOException {
    // Convert Hive filter to Iceberg filter
    String hiveFilter = job.get(TableScanDesc.FILTER_EXPR_CONF_STR);
    if (hiveFilter != null) {
        ExprNodeGenericFuncDesc exprNodeDesc = SerializationUtilities.deserializeObject(hiveFilter, ExprNodeGenericFuncDesc.class);
        SearchArgument sarg = ConvertAstToSearchArg.create(job, exprNodeDesc);
        try {
            Expression filter = HiveIcebergFilterFactory.generateFilterExpression(sarg);
            job.set(InputFormatConfig.FILTER_EXPRESSION, SerializationUtil.serializeToBase64(filter));
        } catch (UnsupportedOperationException e) {
            LOG.warn("Unable to create Iceberg filter, continuing without filter (will be applied by Hive later): ", e);
        }
    }
    job.set(InputFormatConfig.SELECTED_COLUMNS, job.get(ColumnProjectionUtils.READ_COLUMN_NAMES_CONF_STR, ""));
    job.set(InputFormatConfig.AS_OF_TIMESTAMP, job.get(TableScanDesc.AS_OF_TIMESTAMP, "-1"));
    job.set(InputFormatConfig.SNAPSHOT_ID, job.get(TableScanDesc.AS_OF_VERSION, "-1"));
    String location = job.get(InputFormatConfig.TABLE_LOCATION);
    return Arrays.stream(super.getSplits(job, numSplits)).map(split -> new HiveIcebergSplit((IcebergSplit) split, location)).toArray(InputSplit[]::new);
}

Example 18 with Expression

use of org.apache.iceberg.expressions.Expression in project metacat by Netflix.

the class IcebergTableHandler method getIcebergTablePartitionMap.

/**
 * get Partition Map.
 *
 * @param tableName         Qualified table name
 * @param partitionsRequest partitionsRequest
 * @param icebergTable      iceberg Table
 * @return partition map
 */
public Map<String, ScanSummary.PartitionMetrics> getIcebergTablePartitionMap(final QualifiedName tableName, final PartitionListRequest partitionsRequest, final Table icebergTable) {
    final long start = this.registry.clock().wallTime();
    final Map<String, ScanSummary.PartitionMetrics> result;
    try {
        if (!Strings.isNullOrEmpty(partitionsRequest.getFilter())) {
            final IcebergFilterGenerator icebergFilterGenerator = new IcebergFilterGenerator(icebergTable.schema().columns());
            final Expression filter = (Expression) new PartitionParser(new StringReader(partitionsRequest.getFilter())).filter().jjtAccept(icebergFilterGenerator, null);
            result = this.icebergTableOpWrapper.getPartitionMetricsMap(icebergTable, filter);
        } else {
            result = this.icebergTableOpWrapper.getPartitionMetricsMap(icebergTable, null);
        }
    } catch (ParseException ex) {
        log.error("Iceberg filter parse error: ", ex);
        throw new IllegalArgumentException(String.format("Iceberg filter parse error. Ex: %s", ex.getMessage()));
    } catch (IllegalStateException e) {
        registry.counter(registry.createId(IcebergRequestMetrics.CounterGetPartitionsExceedThresholdFailure.getMetricName()).withTags(tableName.parts())).increment();
        final String message = String.format("Number of partitions queried for table %s exceeded the threshold %d", tableName, connectorContext.getConfig().getMaxPartitionsThreshold());
        log.warn(message);
        throw new IllegalArgumentException(message);
    } finally {
        final long duration = registry.clock().wallTime() - start;
        log.info("Time taken to getIcebergTablePartitionMap {} is {} ms", tableName, duration);
        this.recordTimer(IcebergRequestMetrics.TagGetPartitionMap.getMetricName(), duration);
        this.increaseCounter(IcebergRequestMetrics.TagGetPartitionMap.getMetricName(), tableName);
    }
    return result;
}

Example 19 with Expression

use of org.apache.iceberg.expressions.Expression in project presto by prestodb.

the class ExpressionConverter method toIcebergExpression.

public static Expression toIcebergExpression(TupleDomain<IcebergColumnHandle> tupleDomain) {
    if (tupleDomain.isAll()) {
        return alwaysTrue();
    }
    if (!tupleDomain.getDomains().isPresent()) {
        return alwaysFalse();
    }
    Map<IcebergColumnHandle, Domain> domainMap = tupleDomain.getDomains().get();
    Expression expression = alwaysTrue();
    for (Map.Entry<IcebergColumnHandle, Domain> entry : domainMap.entrySet()) {
        IcebergColumnHandle columnHandle = entry.getKey();
        Domain domain = entry.getValue();
        expression = and(expression, toIcebergExpression(columnHandle.getName(), columnHandle.getType(), domain));
    }
    return expression;
}

Example 20 with Expression

use of org.apache.iceberg.expressions.Expression in project drill by apache.

the class TestFilterTransformer method testToFilterIsNotNull.

@Test
public void testToFilterIsNotNull() {
    Expression expected = Expressions.notNull(MetastoreColumn.ROW_GROUP_INDEX.columnName());
    Expression actual = transformer.transform(FilterExpression.isNotNull(MetastoreColumn.ROW_GROUP_INDEX));
    assertEquals(expected.toString(), actual.toString());
}