use of io.trino.spi.type.BooleanType.BOOLEAN in project trino by trinodb.
the class DeltaLakeMetadata method finishOptimize.
private void finishOptimize(ConnectorSession session, DeltaLakeTableExecuteHandle executeHandle, Collection<Slice> fragments, List<Object> splitSourceInfo) {
DeltaTableOptimizeHandle optimizeHandle = (DeltaTableOptimizeHandle) executeHandle.getProcedureHandle();
long readVersion = optimizeHandle.getCurrentVersion().orElseThrow(() -> new IllegalArgumentException("currentVersion not set"));
Optional<Long> checkpointInterval = optimizeHandle.getMetadataEntry().getCheckpointInterval();
String tableLocation = executeHandle.getTableLocation();
// paths to be deleted
Set<Path> scannedPaths = splitSourceInfo.stream().map(file -> new Path((String) file)).collect(toImmutableSet());
// files to be added
List<DataFileInfo> dataFileInfos = fragments.stream().map(Slice::getBytes).map(dataFileInfoCodec::fromJson).collect(toImmutableList());
boolean writeCommitted = false;
try {
TransactionLogWriter transactionLogWriter = transactionLogWriterFactory.newWriter(session, tableLocation);
long createdTime = Instant.now().toEpochMilli();
long commitVersion = readVersion + 1;
transactionLogWriter.appendCommitInfoEntry(new CommitInfoEntry(commitVersion, createdTime, session.getUser(), session.getUser(), OPTIMIZE_OPERATION, ImmutableMap.of("queryId", session.getQueryId()), null, null, "trino-" + nodeVersion + "-" + nodeId, readVersion, ISOLATION_LEVEL, true));
// TODO: Delta writes another field "operationMetrics" that I haven't
// seen before. It contains delete/update metrics. Investigate/include it.
long writeTimestamp = Instant.now().toEpochMilli();
for (Path scannedPath : scannedPaths) {
String relativePath = new Path(tableLocation).toUri().relativize(scannedPath.toUri()).toString();
transactionLogWriter.appendRemoveFileEntry(new RemoveFileEntry(relativePath, writeTimestamp, false));
}
// Note: during writes we want to preserve original case of partition columns
List<String> partitionColumns = optimizeHandle.getMetadataEntry().getOriginalPartitionColumns();
appendAddFileEntries(transactionLogWriter, dataFileInfos, partitionColumns, false);
transactionLogWriter.flush();
writeCommitted = true;
writeCheckpointIfNeeded(session, executeHandle.getSchemaTableName(), checkpointInterval, commitVersion);
} catch (Exception e) {
if (!writeCommitted) {
// TODO perhaps it should happen in a background thread
cleanupFailedWrite(session, tableLocation, dataFileInfos);
}
throw new TrinoException(DELTA_LAKE_BAD_WRITE, "Failed to write Delta Lake transaction log entry", e);
}
}
use of io.trino.spi.type.BooleanType.BOOLEAN in project trino by trinodb.
the class TupleDomainParquetPredicate method getDomain.
/**
* Get a domain for the ranges defined by each pair of elements from {@code minimums} and {@code maximums}.
* Both arrays must have the same length.
*/
private static Domain getDomain(ColumnDescriptor column, Type type, List<Object> minimums, List<Object> maximums, boolean hasNullValue, DateTimeZone timeZone) {
checkArgument(minimums.size() == maximums.size(), "Expected minimums and maximums to have the same size");
if (type.equals(BOOLEAN)) {
boolean hasTrueValues = minimums.stream().anyMatch(value -> (boolean) value) || maximums.stream().anyMatch(value -> (boolean) value);
boolean hasFalseValues = minimums.stream().anyMatch(value -> !(boolean) value) || maximums.stream().anyMatch(value -> !(boolean) value);
if (hasTrueValues && hasFalseValues) {
return Domain.all(type);
}
if (hasTrueValues) {
return Domain.create(ValueSet.of(type, true), hasNullValue);
}
if (hasFalseValues) {
return Domain.create(ValueSet.of(type, false), hasNullValue);
}
// All nulls case is handled earlier
throw new VerifyException("Impossible boolean statistics");
}
if (type.equals(BIGINT) || type.equals(INTEGER) || type.equals(DATE) || type.equals(SMALLINT) || type.equals(TINYINT)) {
List<Range> ranges = new ArrayList<>();
for (int i = 0; i < minimums.size(); i++) {
long min = asLong(minimums.get(i));
long max = asLong(maximums.get(i));
if (isStatisticsOverflow(type, min, max)) {
return Domain.create(ValueSet.all(type), hasNullValue);
}
ranges.add(Range.range(type, min, true, max, true));
}
return Domain.create(ValueSet.ofRanges(ranges), hasNullValue);
}
if (type instanceof DecimalType) {
DecimalType decimalType = (DecimalType) type;
List<Range> ranges = new ArrayList<>();
if (decimalType.isShort()) {
for (int i = 0; i < minimums.size(); i++) {
Object min = minimums.get(i);
Object max = maximums.get(i);
long minValue = min instanceof Binary ? getShortDecimalValue(((Binary) min).getBytes()) : asLong(min);
long maxValue = min instanceof Binary ? getShortDecimalValue(((Binary) max).getBytes()) : asLong(max);
if (isStatisticsOverflow(type, minValue, maxValue)) {
return Domain.create(ValueSet.all(type), hasNullValue);
}
ranges.add(Range.range(type, minValue, true, maxValue, true));
}
} else {
for (int i = 0; i < minimums.size(); i++) {
Int128 min = Int128.fromBigEndian(((Binary) minimums.get(i)).getBytes());
Int128 max = Int128.fromBigEndian(((Binary) maximums.get(i)).getBytes());
ranges.add(Range.range(type, min, true, max, true));
}
}
return Domain.create(ValueSet.ofRanges(ranges), hasNullValue);
}
if (type.equals(REAL)) {
List<Range> ranges = new ArrayList<>();
for (int i = 0; i < minimums.size(); i++) {
Float min = (Float) minimums.get(i);
Float max = (Float) maximums.get(i);
if (min.isNaN() || max.isNaN()) {
return Domain.create(ValueSet.all(type), hasNullValue);
}
ranges.add(Range.range(type, (long) floatToRawIntBits(min), true, (long) floatToRawIntBits(max), true));
}
return Domain.create(ValueSet.ofRanges(ranges), hasNullValue);
}
if (type.equals(DOUBLE)) {
List<Range> ranges = new ArrayList<>();
for (int i = 0; i < minimums.size(); i++) {
Double min = (Double) minimums.get(i);
Double max = (Double) maximums.get(i);
if (min.isNaN() || max.isNaN()) {
return Domain.create(ValueSet.all(type), hasNullValue);
}
ranges.add(Range.range(type, min, true, max, true));
}
return Domain.create(ValueSet.ofRanges(ranges), hasNullValue);
}
if (type instanceof VarcharType) {
List<Range> ranges = new ArrayList<>();
for (int i = 0; i < minimums.size(); i++) {
Slice min = Slices.wrappedBuffer(((Binary) minimums.get(i)).toByteBuffer());
Slice max = Slices.wrappedBuffer(((Binary) maximums.get(i)).toByteBuffer());
ranges.add(Range.range(type, min, true, max, true));
}
return Domain.create(ValueSet.ofRanges(ranges), hasNullValue);
}
if (type instanceof TimestampType) {
if (column.getPrimitiveType().getPrimitiveTypeName().equals(INT96)) {
TrinoTimestampEncoder<?> timestampEncoder = createTimestampEncoder((TimestampType) type, timeZone);
List<Object> values = new ArrayList<>();
for (int i = 0; i < minimums.size(); i++) {
Object min = minimums.get(i);
Object max = maximums.get(i);
// available and valid in that special case
if (!(min instanceof Binary) || !(max instanceof Binary) || !min.equals(max)) {
return Domain.create(ValueSet.all(type), hasNullValue);
}
values.add(timestampEncoder.getTimestamp(decodeInt96Timestamp((Binary) min)));
}
return Domain.multipleValues(type, values, hasNullValue);
}
if (column.getPrimitiveType().getPrimitiveTypeName().equals(INT64)) {
LogicalTypeAnnotation logicalTypeAnnotation = column.getPrimitiveType().getLogicalTypeAnnotation();
if (!(logicalTypeAnnotation instanceof TimestampLogicalTypeAnnotation)) {
// Invalid statistics. Unit and UTC adjustment are not known
return Domain.create(ValueSet.all(type), hasNullValue);
}
TimestampLogicalTypeAnnotation timestampTypeAnnotation = (TimestampLogicalTypeAnnotation) logicalTypeAnnotation;
// Bail out if the precision is not known
if (timestampTypeAnnotation.getUnit() == null) {
return Domain.create(ValueSet.all(type), hasNullValue);
}
TrinoTimestampEncoder<?> timestampEncoder = createTimestampEncoder((TimestampType) type, DateTimeZone.UTC);
List<Range> ranges = new ArrayList<>();
for (int i = 0; i < minimums.size(); i++) {
long min = (long) minimums.get(i);
long max = (long) maximums.get(i);
ranges.add(Range.range(type, timestampEncoder.getTimestamp(decodeInt64Timestamp(min, timestampTypeAnnotation.getUnit())), true, timestampEncoder.getTimestamp(decodeInt64Timestamp(max, timestampTypeAnnotation.getUnit())), true));
}
return Domain.create(ValueSet.ofRanges(ranges), hasNullValue);
}
}
return Domain.create(ValueSet.all(type), hasNullValue);
}
use of io.trino.spi.type.BooleanType.BOOLEAN in project trino by trinodb.
the class QueryPlanner method planExpand.
public RelationPlan planExpand(Query query) {
checkArgument(analysis.isExpandableQuery(query), "query is not registered as expandable");
Union union = (Union) query.getQueryBody();
ImmutableList.Builder<NodeAndMappings> recursionSteps = ImmutableList.builder();
// plan anchor relation
Relation anchorNode = union.getRelations().get(0);
RelationPlan anchorPlan = new RelationPlanner(analysis, symbolAllocator, idAllocator, lambdaDeclarationToSymbolMap, plannerContext, outerContext, session, recursiveSubqueries).process(anchorNode, null);
// prune anchor plan outputs to contain only the symbols exposed in the scope
NodeAndMappings prunedAnchorPlan = pruneInvisibleFields(anchorPlan, idAllocator);
// if the anchor plan has duplicate output symbols, add projection on top to make the symbols unique
// This is necessary to successfully unroll recursion: the recursion step relation must follow
// the same layout while it might not have duplicate outputs where the anchor plan did
NodeAndMappings disambiguatedAnchorPlan = disambiguateOutputs(prunedAnchorPlan, symbolAllocator, idAllocator);
anchorPlan = new RelationPlan(disambiguatedAnchorPlan.getNode(), analysis.getScope(query), disambiguatedAnchorPlan.getFields(), outerContext);
recursionSteps.add(copy(anchorPlan.getRoot(), anchorPlan.getFieldMappings()));
// plan recursion step
Relation recursionStepRelation = union.getRelations().get(1);
RelationPlan recursionStepPlan = new RelationPlanner(analysis, symbolAllocator, idAllocator, lambdaDeclarationToSymbolMap, plannerContext, outerContext, session, ImmutableMap.of(NodeRef.of(analysis.getRecursiveReference(query)), anchorPlan)).process(recursionStepRelation, null);
// coerce recursion step outputs and prune them to contain only the symbols exposed in the scope
NodeAndMappings coercedRecursionStep;
List<Type> types = analysis.getRelationCoercion(recursionStepRelation);
if (types == null) {
coercedRecursionStep = pruneInvisibleFields(recursionStepPlan, idAllocator);
} else {
coercedRecursionStep = coerce(recursionStepPlan, types, symbolAllocator, idAllocator);
}
NodeAndMappings replacementSpot = new NodeAndMappings(anchorPlan.getRoot(), anchorPlan.getFieldMappings());
PlanNode recursionStep = coercedRecursionStep.getNode();
List<Symbol> mappings = coercedRecursionStep.getFields();
// unroll recursion
int maxRecursionDepth = getMaxRecursionDepth(session);
for (int i = 0; i < maxRecursionDepth; i++) {
recursionSteps.add(copy(recursionStep, mappings));
NodeAndMappings replacement = copy(recursionStep, mappings);
// if the recursion step plan has duplicate output symbols, add projection on top to make the symbols unique
// This is necessary to successfully unroll recursion: the relation on the next recursion step must follow
// the same layout while it might not have duplicate outputs where the plan for this step did
replacement = disambiguateOutputs(replacement, symbolAllocator, idAllocator);
recursionStep = replace(recursionStep, replacementSpot, replacement);
replacementSpot = replacement;
}
// after the last recursion step, check if the recursion converged. the last step is expected to return empty result
// 1. append window to count rows
NodeAndMappings checkConvergenceStep = copy(recursionStep, mappings);
Symbol countSymbol = symbolAllocator.newSymbol("count", BIGINT);
ResolvedFunction function = plannerContext.getMetadata().resolveFunction(session, QualifiedName.of("count"), ImmutableList.of());
WindowNode.Function countFunction = new WindowNode.Function(function, ImmutableList.of(), DEFAULT_FRAME, false);
WindowNode windowNode = new WindowNode(idAllocator.getNextId(), checkConvergenceStep.getNode(), new WindowNode.Specification(ImmutableList.of(), Optional.empty()), ImmutableMap.of(countSymbol, countFunction), Optional.empty(), ImmutableSet.of(), 0);
// 2. append filter to fail on non-empty result
ResolvedFunction fail = plannerContext.getMetadata().resolveFunction(session, QualifiedName.of("fail"), fromTypes(VARCHAR));
String recursionLimitExceededMessage = format("Recursion depth limit exceeded (%s). Use 'max_recursion_depth' session property to modify the limit.", maxRecursionDepth);
Expression predicate = new IfExpression(new ComparisonExpression(GREATER_THAN_OR_EQUAL, countSymbol.toSymbolReference(), new GenericLiteral("BIGINT", "0")), new Cast(new FunctionCall(fail.toQualifiedName(), ImmutableList.of(new Cast(new StringLiteral(recursionLimitExceededMessage), toSqlType(VARCHAR)))), toSqlType(BOOLEAN)), TRUE_LITERAL);
FilterNode filterNode = new FilterNode(idAllocator.getNextId(), windowNode, predicate);
recursionSteps.add(new NodeAndMappings(filterNode, checkConvergenceStep.getFields()));
// union all the recursion steps
List<NodeAndMappings> recursionStepsToUnion = recursionSteps.build();
List<Symbol> unionOutputSymbols = anchorPlan.getFieldMappings().stream().map(symbol -> symbolAllocator.newSymbol(symbol, "_expanded")).collect(toImmutableList());
ImmutableListMultimap.Builder<Symbol, Symbol> unionSymbolMapping = ImmutableListMultimap.builder();
for (NodeAndMappings plan : recursionStepsToUnion) {
for (int i = 0; i < unionOutputSymbols.size(); i++) {
unionSymbolMapping.put(unionOutputSymbols.get(i), plan.getFields().get(i));
}
}
List<PlanNode> nodesToUnion = recursionStepsToUnion.stream().map(NodeAndMappings::getNode).collect(toImmutableList());
PlanNode result = new UnionNode(idAllocator.getNextId(), nodesToUnion, unionSymbolMapping.build(), unionOutputSymbols);
if (union.isDistinct()) {
result = new AggregationNode(idAllocator.getNextId(), result, ImmutableMap.of(), singleGroupingSet(result.getOutputSymbols()), ImmutableList.of(), AggregationNode.Step.SINGLE, Optional.empty(), Optional.empty());
}
return new RelationPlan(result, anchorPlan.getScope(), unionOutputSymbols, outerContext);
}
use of io.trino.spi.type.BooleanType.BOOLEAN in project trino by trinodb.
the class PhoenixClient method beginCreateTable.
@Override
public JdbcOutputTableHandle beginCreateTable(ConnectorSession session, ConnectorTableMetadata tableMetadata) {
SchemaTableName schemaTableName = tableMetadata.getTable();
String schema = schemaTableName.getSchemaName();
String table = schemaTableName.getTableName();
if (!getSchemaNames(session).contains(schema)) {
throw new SchemaNotFoundException(schema);
}
try (Connection connection = connectionFactory.openConnection(session)) {
ConnectorIdentity identity = session.getIdentity();
schema = getIdentifierMapping().toRemoteSchemaName(identity, connection, schema);
table = getIdentifierMapping().toRemoteTableName(identity, connection, schema, table);
schema = toPhoenixSchemaName(schema);
LinkedList<ColumnMetadata> tableColumns = new LinkedList<>(tableMetadata.getColumns());
Map<String, Object> tableProperties = tableMetadata.getProperties();
Optional<Boolean> immutableRows = PhoenixTableProperties.getImmutableRows(tableProperties);
String immutable = immutableRows.isPresent() && immutableRows.get() ? "IMMUTABLE" : "";
ImmutableList.Builder<String> columnNames = ImmutableList.builder();
ImmutableList.Builder<Type> columnTypes = ImmutableList.builder();
ImmutableList.Builder<String> columnList = ImmutableList.builder();
Set<ColumnMetadata> rowkeyColumns = tableColumns.stream().filter(col -> isPrimaryKey(col, tableProperties)).collect(toSet());
ImmutableList.Builder<String> pkNames = ImmutableList.builder();
Optional<String> rowkeyColumn = Optional.empty();
if (rowkeyColumns.isEmpty()) {
// Add a rowkey when not specified in DDL
columnList.add(ROWKEY + " bigint not null");
pkNames.add(ROWKEY);
execute(session, format("CREATE SEQUENCE %s", getEscapedTableName(schema, table + "_sequence")));
rowkeyColumn = Optional.of(ROWKEY);
}
for (ColumnMetadata column : tableColumns) {
String columnName = getIdentifierMapping().toRemoteColumnName(connection, column.getName());
columnNames.add(columnName);
columnTypes.add(column.getType());
String typeStatement = toWriteMapping(session, column.getType()).getDataType();
if (rowkeyColumns.contains(column)) {
typeStatement += " not null";
pkNames.add(columnName);
}
columnList.add(format("%s %s", getEscapedArgument(columnName), typeStatement));
}
ImmutableList.Builder<String> tableOptions = ImmutableList.builder();
PhoenixTableProperties.getSaltBuckets(tableProperties).ifPresent(value -> tableOptions.add(TableProperty.SALT_BUCKETS + "=" + value));
PhoenixTableProperties.getSplitOn(tableProperties).ifPresent(value -> tableOptions.add("SPLIT ON (" + value.replace('"', '\'') + ")"));
PhoenixTableProperties.getDisableWal(tableProperties).ifPresent(value -> tableOptions.add(TableProperty.DISABLE_WAL + "=" + value));
PhoenixTableProperties.getDefaultColumnFamily(tableProperties).ifPresent(value -> tableOptions.add(TableProperty.DEFAULT_COLUMN_FAMILY + "=" + value));
PhoenixTableProperties.getBloomfilter(tableProperties).ifPresent(value -> tableOptions.add(HColumnDescriptor.BLOOMFILTER + "='" + value + "'"));
PhoenixTableProperties.getVersions(tableProperties).ifPresent(value -> tableOptions.add(HConstants.VERSIONS + "=" + value));
PhoenixTableProperties.getMinVersions(tableProperties).ifPresent(value -> tableOptions.add(HColumnDescriptor.MIN_VERSIONS + "=" + value));
PhoenixTableProperties.getCompression(tableProperties).ifPresent(value -> tableOptions.add(HColumnDescriptor.COMPRESSION + "='" + value + "'"));
PhoenixTableProperties.getTimeToLive(tableProperties).ifPresent(value -> tableOptions.add(HColumnDescriptor.TTL + "=" + value));
PhoenixTableProperties.getDataBlockEncoding(tableProperties).ifPresent(value -> tableOptions.add(HColumnDescriptor.DATA_BLOCK_ENCODING + "='" + value + "'"));
String sql = format("CREATE %s TABLE %s (%s , CONSTRAINT PK PRIMARY KEY (%s)) %s", immutable, getEscapedTableName(schema, table), join(", ", columnList.build()), join(", ", pkNames.build()), join(", ", tableOptions.build()));
execute(session, sql);
return new PhoenixOutputTableHandle(schema, table, columnNames.build(), columnTypes.build(), Optional.empty(), rowkeyColumn);
} catch (SQLException e) {
if (e.getErrorCode() == SQLExceptionCode.TABLE_ALREADY_EXIST.getErrorCode()) {
throw new TrinoException(ALREADY_EXISTS, "Phoenix table already exists", e);
}
throw new TrinoException(PHOENIX_METADATA_ERROR, "Error creating Phoenix table", e);
}
}
use of io.trino.spi.type.BooleanType.BOOLEAN in project trino by trinodb.
the class MongoSession method guessFieldType.
private Optional<TypeSignature> guessFieldType(Object value) {
if (value == null) {
return Optional.empty();
}
TypeSignature typeSignature = null;
if (value instanceof String) {
typeSignature = createUnboundedVarcharType().getTypeSignature();
}
if (value instanceof Binary) {
typeSignature = VARBINARY.getTypeSignature();
} else if (value instanceof Integer || value instanceof Long) {
typeSignature = BIGINT.getTypeSignature();
} else if (value instanceof Boolean) {
typeSignature = BOOLEAN.getTypeSignature();
} else if (value instanceof Float || value instanceof Double) {
typeSignature = DOUBLE.getTypeSignature();
} else if (value instanceof Date) {
typeSignature = TIMESTAMP_MILLIS.getTypeSignature();
} else if (value instanceof ObjectId) {
typeSignature = OBJECT_ID.getTypeSignature();
} else if (value instanceof List) {
List<Optional<TypeSignature>> subTypes = ((List<?>) value).stream().map(this::guessFieldType).collect(toList());
if (subTypes.isEmpty() || subTypes.stream().anyMatch(Optional::isEmpty)) {
return Optional.empty();
}
Set<TypeSignature> signatures = subTypes.stream().map(Optional::get).collect(toSet());
if (signatures.size() == 1) {
typeSignature = new TypeSignature(StandardTypes.ARRAY, signatures.stream().map(TypeSignatureParameter::typeParameter).collect(Collectors.toList()));
} else {
// TODO: client doesn't handle empty field name row type yet
typeSignature = new TypeSignature(StandardTypes.ROW, IntStream.range(0, subTypes.size()).mapToObj(idx -> TypeSignatureParameter.namedTypeParameter(new NamedTypeSignature(Optional.of(new RowFieldName(format("%s%d", implicitPrefix, idx + 1))), subTypes.get(idx).get()))).collect(toList()));
}
} else if (value instanceof Document) {
List<TypeSignatureParameter> parameters = new ArrayList<>();
for (String key : ((Document) value).keySet()) {
Optional<TypeSignature> fieldType = guessFieldType(((Document) value).get(key));
if (fieldType.isPresent()) {
parameters.add(TypeSignatureParameter.namedTypeParameter(new NamedTypeSignature(Optional.of(new RowFieldName(key)), fieldType.get())));
}
}
if (!parameters.isEmpty()) {
typeSignature = new TypeSignature(StandardTypes.ROW, parameters);
}
} else if (value instanceof DBRef) {
List<TypeSignatureParameter> parameters = new ArrayList<>();
TypeSignature idFieldType = guessFieldType(((DBRef) value).getId()).orElseThrow(() -> new UnsupportedOperationException("Unable to guess $id field type of DBRef from: " + ((DBRef) value).getId()));
parameters.add(TypeSignatureParameter.namedTypeParameter(new NamedTypeSignature(Optional.of(new RowFieldName(DATABASE_NAME)), VARCHAR.getTypeSignature())));
parameters.add(TypeSignatureParameter.namedTypeParameter(new NamedTypeSignature(Optional.of(new RowFieldName(COLLECTION_NAME)), VARCHAR.getTypeSignature())));
parameters.add(TypeSignatureParameter.namedTypeParameter(new NamedTypeSignature(Optional.of(new RowFieldName(ID)), idFieldType)));
typeSignature = new TypeSignature(StandardTypes.ROW, parameters);
}
return Optional.ofNullable(typeSignature);
}
Aggregations