use of org.apache.asterix.om.types.ARecordType in project asterixdb by apache.
the class RTreeAccessMethod method createSecondaryToPrimaryPlan.
private ILogicalOperator createSecondaryToPrimaryPlan(OptimizableOperatorSubTree indexSubTree, OptimizableOperatorSubTree probeSubTree, Index chosenIndex, AccessMethodAnalysisContext analysisCtx, boolean retainInput, boolean retainNull, boolean requiresBroadcast, IOptimizationContext context) throws AlgebricksException {
IOptimizableFuncExpr optFuncExpr = AccessMethodUtils.chooseFirstOptFuncExpr(chosenIndex, analysisCtx);
Dataset dataset = indexSubTree.getDataset();
ARecordType recordType = indexSubTree.getRecordType();
ARecordType metaRecordType = indexSubTree.getMetaRecordType();
int optFieldIdx = AccessMethodUtils.chooseFirstOptFuncVar(chosenIndex, analysisCtx);
Pair<IAType, Boolean> keyPairType = Index.getNonNullableOpenFieldType(optFuncExpr.getFieldType(optFieldIdx), optFuncExpr.getFieldName(optFieldIdx), recordType);
if (keyPairType == null) {
return null;
}
// Get the number of dimensions corresponding to the field indexed by chosenIndex.
IAType spatialType = keyPairType.first;
int numDimensions = NonTaggedFormatUtil.getNumDimensions(spatialType.getTypeTag());
int numSecondaryKeys = numDimensions * 2;
// we made sure indexSubTree has datasource scan
AbstractDataSourceOperator dataSourceOp = (AbstractDataSourceOperator) indexSubTree.getDataSourceRef().getValue();
RTreeJobGenParams jobGenParams = new RTreeJobGenParams(chosenIndex.getIndexName(), IndexType.RTREE, dataset.getDataverseName(), dataset.getDatasetName(), retainInput, requiresBroadcast);
// A spatial object is serialized in the constant of the func expr we are optimizing.
// The R-Tree expects as input an MBR represented with 1 field per dimension.
// Here we generate vars and funcs for extracting MBR fields from the constant into fields of a tuple (as the
// R-Tree expects them).
// List of variables for the assign.
ArrayList<LogicalVariable> keyVarList = new ArrayList<>();
// List of expressions for the assign.
ArrayList<Mutable<ILogicalExpression>> keyExprList = new ArrayList<>();
Pair<ILogicalExpression, Boolean> returnedSearchKeyExpr = AccessMethodUtils.createSearchKeyExpr(optFuncExpr, indexSubTree, probeSubTree);
ILogicalExpression searchKeyExpr = returnedSearchKeyExpr.first;
for (int i = 0; i < numSecondaryKeys; i++) {
// The create MBR function "extracts" one field of an MBR around the given spatial object.
AbstractFunctionCallExpression createMBR = new ScalarFunctionCallExpression(FunctionUtil.getFunctionInfo(BuiltinFunctions.CREATE_MBR));
// Spatial object is the constant from the func expr we are optimizing.
createMBR.getArguments().add(new MutableObject<>(searchKeyExpr));
// The number of dimensions.
createMBR.getArguments().add(new MutableObject<ILogicalExpression>(new ConstantExpression(new AsterixConstantValue(new AInt32(numDimensions)))));
// Which part of the MBR to extract.
createMBR.getArguments().add(new MutableObject<ILogicalExpression>(new ConstantExpression(new AsterixConstantValue(new AInt32(i)))));
// Add a variable and its expr to the lists which will be passed into an assign op.
LogicalVariable keyVar = context.newVar();
keyVarList.add(keyVar);
keyExprList.add(new MutableObject<ILogicalExpression>(createMBR));
}
jobGenParams.setKeyVarList(keyVarList);
// Assign operator that "extracts" the MBR fields from the func-expr constant into a tuple.
AssignOperator assignSearchKeys = new AssignOperator(keyVarList, keyExprList);
if (probeSubTree == null) {
// We are optimizing a selection query.
// Input to this assign is the EmptyTupleSource (which the dataSourceScan also must have had as input).
assignSearchKeys.getInputs().add(new MutableObject<>(OperatorManipulationUtil.deepCopy(dataSourceOp.getInputs().get(0).getValue())));
assignSearchKeys.setExecutionMode(dataSourceOp.getExecutionMode());
} else {
// We are optimizing a join, place the assign op top of the probe subtree.
assignSearchKeys.getInputs().add(probeSubTree.getRootRef());
}
ILogicalOperator secondaryIndexUnnestOp = AccessMethodUtils.createSecondaryIndexUnnestMap(dataset, recordType, metaRecordType, chosenIndex, assignSearchKeys, jobGenParams, context, false, retainInput, retainNull);
// Generate the rest of the upstream plan which feeds the search results into the primary index.
return dataset.getDatasetType() == DatasetType.EXTERNAL ? AccessMethodUtils.createExternalDataLookupUnnestMap(dataSourceOp, dataset, recordType, secondaryIndexUnnestOp, context, retainInput, retainNull) : AccessMethodUtils.createPrimaryIndexUnnestMap(dataSourceOp, dataset, recordType, metaRecordType, secondaryIndexUnnestOp, context, true, retainInput, false, false);
}
use of org.apache.asterix.om.types.ARecordType in project asterixdb by apache.
the class IntroduceLSMComponentFilterRule method getFieldNameFromSubAssignTree.
private Pair<ARecordType, List<String>> getFieldNameFromSubAssignTree(IOptimizableFuncExpr optFuncExpr, AbstractLogicalOperator op, int varIndex, ARecordType recType) {
AbstractLogicalExpression expr = null;
if (op.getOperatorTag() == LogicalOperatorTag.ASSIGN) {
AssignOperator assignOp = (AssignOperator) op;
expr = (AbstractLogicalExpression) assignOp.getExpressions().get(varIndex).getValue();
}
if (expr == null || expr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
return null;
}
AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) expr;
FunctionIdentifier funcIdent = funcExpr.getFunctionIdentifier();
if (funcIdent == BuiltinFunctions.FIELD_ACCESS_BY_NAME || funcIdent == BuiltinFunctions.FIELD_ACCESS_BY_INDEX) {
//get the variable from here. Figure out which input it came from. Go to that input!!!
ArrayList<LogicalVariable> usedVars = new ArrayList<>();
expr.getUsedVariables(usedVars);
LogicalVariable usedVar = usedVars.get(0);
List<String> returnList = new ArrayList<>();
//Find the input that it came from
for (int varCheck = 0; varCheck < op.getInputs().size(); varCheck++) {
AbstractLogicalOperator nestedOp = (AbstractLogicalOperator) op.getInputs().get(varCheck).getValue();
if (nestedOp.getOperatorTag() != LogicalOperatorTag.ASSIGN) {
if (varCheck == op.getInputs().size() - 1) {
}
} else {
int nestedAssignVar = ((AssignOperator) nestedOp).getVariables().indexOf(usedVar);
if (nestedAssignVar == -1) {
continue;
}
//get the nested info from the lower input
Pair<ARecordType, List<String>> lowerInfo = getFieldNameFromSubAssignTree(optFuncExpr, (AbstractLogicalOperator) op.getInputs().get(varCheck).getValue(), nestedAssignVar, recType);
if (lowerInfo != null) {
recType = lowerInfo.first;
returnList = lowerInfo.second;
}
}
}
if (funcIdent == BuiltinFunctions.FIELD_ACCESS_BY_NAME) {
String fieldName = ConstantExpressionUtil.getStringArgument(funcExpr, 1);
if (fieldName == null) {
return null;
}
returnList.add(fieldName);
return new Pair<>(recType, returnList);
} else if (funcIdent == BuiltinFunctions.FIELD_ACCESS_BY_INDEX) {
Integer fieldIndex = ConstantExpressionUtil.getIntArgument(funcExpr, 1);
if (fieldIndex == null) {
return null;
}
returnList.add(recType.getFieldNames()[fieldIndex]);
IAType subType = recType.getFieldTypes()[fieldIndex];
if (subType.getTypeTag() == ATypeTag.OBJECT) {
recType = (ARecordType) subType;
}
return new Pair<>(recType, returnList);
}
}
ILogicalExpression argExpr = funcExpr.getArguments().get(0).getValue();
if (argExpr.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
return null;
}
return null;
}
use of org.apache.asterix.om.types.ARecordType in project asterixdb by apache.
the class IntroduceLSMComponentFilterRule method findMacthedExprFieldName.
private boolean findMacthedExprFieldName(IOptimizableFuncExpr optFuncExpr, AbstractLogicalOperator op, Dataset dataset, ARecordType recType, List<Index> datasetIndexes, IOptimizationContext context) throws AlgebricksException {
AbstractLogicalOperator descendantOp = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
while (descendantOp != null) {
if (descendantOp.getOperatorTag() == LogicalOperatorTag.ASSIGN) {
AssignOperator assignOp = (AssignOperator) descendantOp;
List<LogicalVariable> varList = assignOp.getVariables();
for (int varIndex = 0; varIndex < varList.size(); varIndex++) {
LogicalVariable var = varList.get(varIndex);
int funcVarIndex = optFuncExpr.findLogicalVar(var);
if (funcVarIndex == -1) {
continue;
}
List<String> fieldName = getFieldNameFromSubAssignTree(optFuncExpr, descendantOp, varIndex, recType).second;
if (fieldName == null) {
return false;
}
optFuncExpr.setFieldName(funcVarIndex, fieldName);
return true;
}
} else if (descendantOp.getOperatorTag() == LogicalOperatorTag.DATASOURCESCAN) {
DataSourceScanOperator scanOp = (DataSourceScanOperator) descendantOp;
List<LogicalVariable> varList = scanOp.getVariables();
for (int varIndex = 0; varIndex < varList.size(); varIndex++) {
LogicalVariable var = varList.get(varIndex);
int funcVarIndex = optFuncExpr.findLogicalVar(var);
if (funcVarIndex == -1) {
continue;
}
// The variable value is one of the partitioning fields.
List<String> fieldName = dataset.getPrimaryKeys().get(varIndex);
if (fieldName == null) {
return false;
}
optFuncExpr.setFieldName(funcVarIndex, fieldName);
return true;
}
} else if (descendantOp.getOperatorTag() == LogicalOperatorTag.UNNEST_MAP) {
UnnestMapOperator unnestMapOp = (UnnestMapOperator) descendantOp;
List<LogicalVariable> varList = unnestMapOp.getVariables();
for (int varIndex = 0; varIndex < varList.size(); varIndex++) {
LogicalVariable var = varList.get(varIndex);
int funcVarIndex = optFuncExpr.findLogicalVar(var);
if (funcVarIndex == -1) {
continue;
}
String indexName;
Index index = null;
ILogicalExpression unnestExpr = unnestMapOp.getExpressionRef().getValue();
if (unnestExpr.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
AbstractFunctionCallExpression f = (AbstractFunctionCallExpression) unnestExpr;
FunctionIdentifier fid = f.getFunctionIdentifier();
if (!fid.equals(BuiltinFunctions.INDEX_SEARCH)) {
throw new IllegalStateException();
}
AccessMethodJobGenParams jobGenParams = new AccessMethodJobGenParams();
jobGenParams.readFromFuncArgs(f.getArguments());
indexName = jobGenParams.indexName;
for (Index idx : datasetIndexes) {
if (idx.getIndexName().compareTo(indexName) == 0) {
index = idx;
break;
}
}
}
IAType metaItemType = ((MetadataProvider) context.getMetadataProvider()).findType(dataset.getMetaItemTypeDataverseName(), dataset.getMetaItemTypeName());
ARecordType metaRecType = (ARecordType) metaItemType;
int numSecondaryKeys = KeyFieldTypeUtil.getNumSecondaryKeys(index, recType, metaRecType);
List<String> fieldName;
if (varIndex >= numSecondaryKeys) {
fieldName = dataset.getPrimaryKeys().get(varIndex - numSecondaryKeys);
} else {
fieldName = index.getKeyFieldNames().get(varIndex);
}
if (fieldName == null) {
return false;
}
optFuncExpr.setFieldName(funcVarIndex, fieldName);
return true;
}
}
if (descendantOp.getInputs().isEmpty()) {
break;
}
descendantOp = (AbstractLogicalOperator) descendantOp.getInputs().get(0).getValue();
}
return false;
}
use of org.apache.asterix.om.types.ARecordType in project asterixdb by apache.
the class JObjectUtil method mergeRecordTypes.
private static ARecordType mergeRecordTypes(ARecordType recType1, ARecordType recType2) {
String[] fieldNames = new String[recType1.getFieldNames().length + recType2.getFieldNames().length];
IAType[] fieldTypes = new IAType[recType1.getFieldTypes().length + recType2.getFieldTypes().length];
int i = 0;
for (; i < recType1.getFieldNames().length; i++) {
fieldNames[i] = recType1.getFieldNames()[i];
fieldTypes[i] = recType1.getFieldTypes()[i];
}
for (int j = 0; j < recType2.getFieldNames().length; i++, j++) {
fieldNames[i] = recType2.getFieldNames()[j];
fieldTypes[i] = recType2.getFieldTypes()[j];
}
return new ARecordType(null, fieldNames, fieldTypes, true);
}
use of org.apache.asterix.om.types.ARecordType in project asterixdb by apache.
the class JObjectUtil method getJType.
public static IJObject getJType(ATypeTag typeTag, IAType type, ByteArrayAccessibleDataInputStream dis, IObjectPool<IJObject, IAType> objectPool) throws IOException {
IJObject jObject;
switch(typeTag) {
case INTEGER:
{
int v = dis.readInt();
jObject = objectPool.allocate(BuiltinType.AINT32);
((JInt) jObject).setValue(v);
break;
}
case FLOAT:
{
float v = dis.readFloat();
jObject = objectPool.allocate(BuiltinType.AFLOAT);
((JFloat) jObject).setValue(v);
break;
}
case DOUBLE:
{
double value = dis.readDouble();
jObject = objectPool.allocate(BuiltinType.ADOUBLE);
((JDouble) jObject).setValue(value);
break;
}
case STRING:
{
String v = dis.readUTF();
jObject = objectPool.allocate(BuiltinType.ASTRING);
((JString) jObject).setValue(v);
break;
}
case BOOLEAN:
jObject = objectPool.allocate(BuiltinType.ABOOLEAN);
((JBoolean) jObject).setValue(dis.readBoolean());
break;
case DATE:
{
int d = dis.readInt();
jObject = objectPool.allocate(BuiltinType.ADATE);
((JDate) jObject).setValue(d);
break;
}
case DATETIME:
{
jObject = objectPool.allocate(BuiltinType.ADATETIME);
long value = dis.readLong();
((JDateTime) jObject).setValue(value);
break;
}
case DURATION:
{
jObject = objectPool.allocate(BuiltinType.ADURATION);
int months = dis.readInt();
long msecs = dis.readLong();
((JDuration) jObject).setValue(months, msecs);
break;
}
case TIME:
{
jObject = objectPool.allocate(BuiltinType.ATIME);
int time = dis.readInt();
((JTime) jObject).setValue(time);
break;
}
case INTERVAL:
{
jObject = objectPool.allocate(BuiltinType.AINTERVAL);
long start = dis.readLong();
long end = dis.readLong();
byte intervalType = dis.readByte();
((JInterval) jObject).setValue(start, end, intervalType);
break;
}
case CIRCLE:
{
jObject = objectPool.allocate(BuiltinType.ACIRCLE);
double x = dis.readDouble();
double y = dis.readDouble();
double radius = dis.readDouble();
JPoint jpoint = (JPoint) objectPool.allocate(BuiltinType.APOINT);
jpoint.setValue(x, y);
((JCircle) jObject).setValue(jpoint, radius);
break;
}
case POINT:
{
jObject = objectPool.allocate(BuiltinType.APOINT);
double x = dis.readDouble();
double y = dis.readDouble();
((JPoint) jObject).setValue(x, y);
break;
}
case POINT3D:
{
jObject = objectPool.allocate(BuiltinType.APOINT3D);
double x = dis.readDouble();
double y = dis.readDouble();
double z = dis.readDouble();
((JPoint3D) jObject).setValue(x, y, z);
break;
}
case LINE:
{
jObject = objectPool.allocate(BuiltinType.ALINE);
double x1 = dis.readDouble();
double y1 = dis.readDouble();
double x2 = dis.readDouble();
double y2 = dis.readDouble();
JPoint jpoint1 = (JPoint) objectPool.allocate(BuiltinType.APOINT);
jpoint1.setValue(x1, y1);
JPoint jpoint2 = (JPoint) objectPool.allocate(BuiltinType.APOINT);
jpoint2.setValue(x2, y2);
((JLine) jObject).setValue(jpoint1, jpoint2);
break;
}
case POLYGON:
{
jObject = objectPool.allocate(BuiltinType.APOLYGON);
short numberOfPoints = dis.readShort();
List<JPoint> points = new ArrayList<JPoint>();
for (int i = 0; i < numberOfPoints; i++) {
JPoint p1 = (JPoint) objectPool.allocate(BuiltinType.APOINT);
p1.setValue(dis.readDouble(), dis.readDouble());
points.add(p1);
}
((JPolygon) jObject).setValue(points.toArray(new APoint[] {}));
break;
}
case RECTANGLE:
{
jObject = objectPool.allocate(BuiltinType.ARECTANGLE);
double x1 = dis.readDouble();
double y1 = dis.readDouble();
double x2 = dis.readDouble();
double y2 = dis.readDouble();
JPoint jpoint1 = (JPoint) objectPool.allocate(BuiltinType.APOINT);
jpoint1.setValue(x1, y1);
JPoint jpoint2 = (JPoint) objectPool.allocate(BuiltinType.APOINT);
jpoint2.setValue(x2, y2);
((JRectangle) jObject).setValue(jpoint1, jpoint2);
break;
}
case MULTISET:
{
AUnorderedListType listType = (AUnorderedListType) type;
IAType elementType = listType.getItemType();
jObject = objectPool.allocate(listType);
boolean fixedSize = false;
ATypeTag tag = EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(dis.readByte());
switch(tag) {
case STRING:
case OBJECT:
case ARRAY:
case MULTISET:
case ANY:
fixedSize = false;
break;
default:
fixedSize = true;
break;
}
// list size
dis.readInt();
int numberOfitems;
numberOfitems = dis.readInt();
if (numberOfitems <= 0) {
break;
}
if (!fixedSize) {
for (int i = 0; i < numberOfitems; i++) {
dis.readInt();
}
}
for (int i = 0; i < numberOfitems; i++) {
IJObject v = getJType(elementType.getTypeTag(), elementType, dis, objectPool);
((JUnorderedList) jObject).add(v);
}
break;
}
case ARRAY:
{
AOrderedListType listType = (AOrderedListType) type;
IAType elementType = listType.getItemType();
jObject = objectPool.allocate(listType);
boolean fixedSize = false;
ATypeTag tag = EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(dis.readByte());
switch(tag) {
case STRING:
case OBJECT:
case ARRAY:
case MULTISET:
case ANY:
fixedSize = false;
break;
default:
fixedSize = true;
break;
}
// list size
dis.readInt();
int numberOfitems;
numberOfitems = dis.readInt();
if (numberOfitems <= 0) {
break;
}
if (!fixedSize) {
for (int i = 0; i < numberOfitems; i++) {
dis.readInt();
}
}
for (int i = 0; i < numberOfitems; i++) {
IJObject v = getJType(elementType.getTypeTag(), elementType, dis, objectPool);
((JOrderedList) jObject).add(v);
}
break;
}
case OBJECT:
ARecordType recordType = (ARecordType) type;
int numberOfSchemaFields = recordType.getFieldTypes().length;
byte[] recordBits = dis.getInputStream().getArray();
boolean isExpanded = false;
dis.getInputStream();
int[] fieldOffsets = new int[numberOfSchemaFields];
IJObject[] closedFields = new IJObject[numberOfSchemaFields];
// reading length is not required.
dis.skip(4);
if (recordType.isOpen()) {
isExpanded = dis.readBoolean();
if (isExpanded) {
dis.readInt();
} else {
}
} else {
}
if (numberOfSchemaFields > 0) {
dis.readInt();
int nullBitMapOffset = 0;
boolean hasOptionalFields = NonTaggedFormatUtil.hasOptionalField(recordType);
if (hasOptionalFields) {
nullBitMapOffset = dis.getInputStream().getPosition();
dis.getInputStream();
} else {
dis.getInputStream();
}
for (int i = 0; i < numberOfSchemaFields; i++) {
fieldOffsets[i] = dis.readInt();
}
for (int fieldNumber = 0; fieldNumber < numberOfSchemaFields; fieldNumber++) {
if (hasOptionalFields) {
byte b1 = recordBits[nullBitMapOffset + fieldNumber / 8];
int p = 1 << (7 - (fieldNumber % 8));
if ((b1 & p) == 0) {
continue;
}
}
IAType[] fieldTypes = recordType.getFieldTypes();
ATypeTag fieldValueTypeTag = null;
IAType fieldType = fieldTypes[fieldNumber];
if (fieldTypes[fieldNumber].getTypeTag() == ATypeTag.UNION) {
if (((AUnionType) fieldTypes[fieldNumber]).isUnknownableType()) {
fieldType = ((AUnionType) fieldTypes[fieldNumber]).getActualType();
fieldValueTypeTag = fieldType.getTypeTag();
}
} else {
fieldValueTypeTag = fieldTypes[fieldNumber].getTypeTag();
}
closedFields[fieldNumber] = getJType(fieldValueTypeTag, fieldType, dis, objectPool);
}
}
if (isExpanded) {
int numberOfOpenFields = dis.readInt();
String[] fieldNames = new String[numberOfOpenFields];
IAType[] fieldTypes = new IAType[numberOfOpenFields];
IJObject[] openFields = new IJObject[numberOfOpenFields];
for (int i = 0; i < numberOfOpenFields; i++) {
dis.readInt();
dis.readInt();
}
for (int i = 0; i < numberOfOpenFields; i++) {
fieldNames[i] = AStringSerializerDeserializer.INSTANCE.deserialize(dis).getStringValue();
ATypeTag openFieldTypeTag = SerializerDeserializerUtil.deserializeTag(dis);
openFields[i] = getJType(openFieldTypeTag, null, dis, objectPool);
fieldTypes[i] = openFields[i].getIAObject().getType();
}
ARecordType openPartRecType = new ARecordType(null, fieldNames, fieldTypes, true);
if (numberOfSchemaFields > 0) {
ARecordType mergedRecordType = mergeRecordTypes(recordType, openPartRecType);
IJObject[] mergedFields = mergeFields(closedFields, openFields);
jObject = objectPool.allocate(recordType);
return new JRecord(mergedRecordType, mergedFields);
} else {
return new JRecord(recordType, openFields);
}
} else {
return new JRecord(recordType, closedFields);
}
default:
throw new RuntimeDataException(ErrorCode.LIBRARY_JOBJECT_UTIL_ILLEGAL_ARGU_TYPE, typeTag);
}
return jObject;
}
Aggregations