Examples with Index org.apache.asterix.metadata.entities.Index used on opensource projects

Example 1 with Index

use of org.apache.asterix.metadata.entities.Index in project asterixdb by apache.

the class TypeUtil method createEnforcedType.

/**
     * Merges typed index fields with specified recordType, allowing indexed fields to be optional.
     * I.e. the type { "personId":int32, "name": string, "address" : { "street": string } } with typed indexes
     * on age:int32, address.state:string will be merged into type { "personId":int32, "name": string,
     * "age": int32? "address" : { "street": string, "state": string? } } Used by open indexes to enforce
     * the type of an indexed record
     */
public static Pair<ARecordType, ARecordType> createEnforcedType(ARecordType recordType, ARecordType metaType, List<Index> indexes) throws AlgebricksException {
    ARecordType enforcedRecordType = recordType;
    ARecordType enforcedMetaType = metaType;
    for (Index index : indexes) {
        if (!index.isSecondaryIndex() || !index.isEnforcingKeyFileds()) {
            continue;
        }
        if (index.hasMetaFields()) {
            throw new AlgebricksException("Indexing an open field is only supported on the record part");
        }
        for (int i = 0; i < index.getKeyFieldNames().size(); i++) {
            Deque<Pair<ARecordType, String>> nestedTypeStack = new ArrayDeque<>();
            List<String> splits = index.getKeyFieldNames().get(i);
            ARecordType nestedFieldType = enforcedRecordType;
            boolean openRecords = false;
            String bridgeName = nestedFieldType.getTypeName();
            int j;
            // Build the stack for the enforced type
            for (j = 1; j < splits.size(); j++) {
                nestedTypeStack.push(new Pair<>(nestedFieldType, splits.get(j - 1)));
                bridgeName = nestedFieldType.getTypeName();
                nestedFieldType = (ARecordType) enforcedRecordType.getSubFieldType(splits.subList(0, j));
                if (nestedFieldType == null) {
                    openRecords = true;
                    break;
                }
            }
            if (openRecords) {
                // create the smallest record
                enforcedRecordType = new ARecordType(splits.get(splits.size() - 2), new String[] { splits.get(splits.size() - 1) }, new IAType[] { AUnionType.createUnknownableType(index.getKeyFieldTypes().get(i)) }, true);
                // create the open part of the nested field
                for (int k = splits.size() - 3; k > (j - 2); k--) {
                    enforcedRecordType = new ARecordType(splits.get(k), new String[] { splits.get(k + 1) }, new IAType[] { AUnionType.createUnknownableType(enforcedRecordType) }, true);
                }
                // Bridge the gap
                Pair<ARecordType, String> gapPair = nestedTypeStack.pop();
                ARecordType parent = gapPair.first;
                IAType[] parentFieldTypes = ArrayUtils.addAll(parent.getFieldTypes().clone(), new IAType[] { AUnionType.createUnknownableType(enforcedRecordType) });
                enforcedRecordType = new ARecordType(bridgeName, ArrayUtils.addAll(parent.getFieldNames(), enforcedRecordType.getTypeName()), parentFieldTypes, true);
            } else {
                //Schema is closed all the way to the field
                //enforced fields are either null or strongly typed
                Map<String, IAType> recordNameTypesMap = TypeUtil.createRecordNameTypeMap(nestedFieldType);
                // if a an enforced field already exists and the type is correct
                IAType enforcedFieldType = recordNameTypesMap.get(splits.get(splits.size() - 1));
                if (enforcedFieldType != null && enforcedFieldType.getTypeTag() == ATypeTag.UNION && ((AUnionType) enforcedFieldType).isUnknownableType()) {
                    enforcedFieldType = ((AUnionType) enforcedFieldType).getActualType();
                }
                if (enforcedFieldType != null && !ATypeHierarchy.canPromote(enforcedFieldType.getTypeTag(), index.getKeyFieldTypes().get(i).getTypeTag())) {
                    throw new AlgebricksException("Cannot enforce field " + index.getKeyFieldNames().get(i) + " to have type " + index.getKeyFieldTypes().get(i));
                }
                if (enforcedFieldType == null) {
                    recordNameTypesMap.put(splits.get(splits.size() - 1), AUnionType.createUnknownableType(index.getKeyFieldTypes().get(i)));
                }
                enforcedRecordType = new ARecordType(nestedFieldType.getTypeName(), recordNameTypesMap.keySet().toArray(new String[recordNameTypesMap.size()]), recordNameTypesMap.values().toArray(new IAType[recordNameTypesMap.size()]), nestedFieldType.isOpen());
            }
            // Create the enforced type for the nested fields in the schema, from the ground up
            if (!nestedTypeStack.isEmpty()) {
                while (!nestedTypeStack.isEmpty()) {
                    Pair<ARecordType, String> nestedTypePair = nestedTypeStack.pop();
                    ARecordType nestedRecType = nestedTypePair.first;
                    IAType[] nestedRecTypeFieldTypes = nestedRecType.getFieldTypes().clone();
                    nestedRecTypeFieldTypes[nestedRecType.getFieldIndex(nestedTypePair.second)] = enforcedRecordType;
                    enforcedRecordType = new ARecordType(nestedRecType.getTypeName() + "_enforced", nestedRecType.getFieldNames(), nestedRecTypeFieldTypes, nestedRecType.isOpen());
                }
            }
        }
    }
    return new Pair<>(enforcedRecordType, enforcedMetaType);
}

Example 2 with Index

use of org.apache.asterix.metadata.entities.Index in project asterixdb by apache.

the class IndexTupleTranslatorTest method test.

@Test
public void test() throws MetadataException, IOException {
    Integer[] indicators = { 0, 1, null };
    for (Integer indicator : indicators) {
        Map<String, String> compactionPolicyProperties = new HashMap<>();
        compactionPolicyProperties.put("max-mergable-component-size", "1073741824");
        compactionPolicyProperties.put("max-tolerance-component-count", "3");
        InternalDatasetDetails details = new InternalDatasetDetails(FileStructure.BTREE, PartitioningStrategy.HASH, Collections.singletonList(Collections.singletonList("row_id")), Collections.singletonList(Collections.singletonList("row_id")), indicator == null ? null : Collections.singletonList(indicator), Collections.singletonList(BuiltinType.AINT64), false, Collections.emptyList(), false);
        Dataset dataset = new Dataset("test", "d1", "foo", "LogType", "CB", "MetaType", "DEFAULT_NG_ALL_NODES", "prefix", compactionPolicyProperties, details, Collections.emptyMap(), DatasetType.INTERNAL, 115, 0);
        Index index = new Index("test", "d1", "i1", IndexType.BTREE, Collections.singletonList(Collections.singletonList("row_id")), indicator == null ? null : Collections.singletonList(indicator), Collections.singletonList(BuiltinType.AINT64), -1, false, false, 0);
        MetadataNode mockMetadataNode = mock(MetadataNode.class);
        when(mockMetadataNode.getDatatype(any(), anyString(), anyString())).thenReturn(new Datatype("test", "d1", new ARecordType("", new String[] { "row_id" }, new IAType[] { BuiltinType.AINT64 }, true), true));
        when(mockMetadataNode.getDataset(any(), anyString(), anyString())).thenReturn(dataset);
        IndexTupleTranslator idxTranslator = new IndexTupleTranslator(null, mockMetadataNode, true);
        ITupleReference tuple = idxTranslator.getTupleFromMetadataEntity(index);
        Index deserializedIndex = idxTranslator.getMetadataEntityFromTuple(tuple);
        if (indicator == null) {
            Assert.assertEquals(Collections.singletonList(new Integer(0)), deserializedIndex.getKeyFieldSourceIndicators());
        } else {
            Assert.assertEquals(index.getKeyFieldSourceIndicators(), deserializedIndex.getKeyFieldSourceIndicators());
        }
    }
}

Example 3 with Index

use of org.apache.asterix.metadata.entities.Index in project asterixdb by apache.

the class IndexTupleTranslator method getMetadataEntityFromTuple.

@Override
public Index getMetadataEntityFromTuple(ITupleReference frameTuple) throws MetadataException, HyracksDataException {
    byte[] serRecord = frameTuple.getFieldData(INDEX_PAYLOAD_TUPLE_FIELD_INDEX);
    int recordStartOffset = frameTuple.getFieldStart(INDEX_PAYLOAD_TUPLE_FIELD_INDEX);
    int recordLength = frameTuple.getFieldLength(INDEX_PAYLOAD_TUPLE_FIELD_INDEX);
    ByteArrayInputStream stream = new ByteArrayInputStream(serRecord, recordStartOffset, recordLength);
    DataInput in = new DataInputStream(stream);
    ARecord rec = recordSerde.deserialize(in);
    String dvName = ((AString) rec.getValueByPos(MetadataRecordTypes.INDEX_ARECORD_DATAVERSENAME_FIELD_INDEX)).getStringValue();
    String dsName = ((AString) rec.getValueByPos(MetadataRecordTypes.INDEX_ARECORD_DATASETNAME_FIELD_INDEX)).getStringValue();
    String indexName = ((AString) rec.getValueByPos(MetadataRecordTypes.INDEX_ARECORD_INDEXNAME_FIELD_INDEX)).getStringValue();
    IndexType indexStructure = IndexType.valueOf(((AString) rec.getValueByPos(MetadataRecordTypes.INDEX_ARECORD_INDEXSTRUCTURE_FIELD_INDEX)).getStringValue());
    IACursor fieldNameCursor = ((AOrderedList) rec.getValueByPos(MetadataRecordTypes.INDEX_ARECORD_SEARCHKEY_FIELD_INDEX)).getCursor();
    List<List<String>> searchKey = new ArrayList<>();
    AOrderedList fieldNameList;
    while (fieldNameCursor.next()) {
        fieldNameList = (AOrderedList) fieldNameCursor.get();
        IACursor nestedFieldNameCursor = (fieldNameList.getCursor());
        List<String> nestedFieldName = new ArrayList<>();
        while (nestedFieldNameCursor.next()) {
            nestedFieldName.add(((AString) nestedFieldNameCursor.get()).getStringValue());
        }
        searchKey.add(nestedFieldName);
    }
    int indexKeyTypeFieldPos = rec.getType().getFieldIndex(INDEX_SEARCHKEY_TYPE_FIELD_NAME);
    IACursor fieldTypeCursor = new ACollectionCursor();
    if (indexKeyTypeFieldPos > 0) {
        fieldTypeCursor = ((AOrderedList) rec.getValueByPos(indexKeyTypeFieldPos)).getCursor();
    }
    List<IAType> searchKeyType = new ArrayList<>(searchKey.size());
    while (fieldTypeCursor.next()) {
        String typeName = ((AString) fieldTypeCursor.get()).getStringValue();
        IAType fieldType = BuiltinTypeMap.getTypeFromTypeName(metadataNode, jobId, dvName, typeName, false);
        searchKeyType.add(fieldType);
    }
    int isEnforcedFieldPos = rec.getType().getFieldIndex(INDEX_ISENFORCED_FIELD_NAME);
    Boolean isEnforcingKeys = false;
    if (isEnforcedFieldPos > 0) {
        isEnforcingKeys = ((ABoolean) rec.getValueByPos(isEnforcedFieldPos)).getBoolean();
    }
    Boolean isPrimaryIndex = ((ABoolean) rec.getValueByPos(MetadataRecordTypes.INDEX_ARECORD_ISPRIMARY_FIELD_INDEX)).getBoolean();
    int pendingOp = ((AInt32) rec.getValueByPos(MetadataRecordTypes.INDEX_ARECORD_PENDINGOP_FIELD_INDEX)).getIntegerValue();
    // Check if there is a gram length as well.
    int gramLength = -1;
    int gramLenPos = rec.getType().getFieldIndex(GRAM_LENGTH_FIELD_NAME);
    if (gramLenPos >= 0) {
        gramLength = ((AInt32) rec.getValueByPos(gramLenPos)).getIntegerValue();
    }
    // Read a field-source-indicator field.
    List<Integer> keyFieldSourceIndicator = new ArrayList<>();
    int keyFieldSourceIndicatorIndex = rec.getType().getFieldIndex(INDEX_SEARCHKEY_SOURCE_INDICATOR_FIELD_NAME);
    if (keyFieldSourceIndicatorIndex >= 0) {
        IACursor cursor = ((AOrderedList) rec.getValueByPos(keyFieldSourceIndicatorIndex)).getCursor();
        while (cursor.next()) {
            keyFieldSourceIndicator.add((int) ((AInt8) cursor.get()).getByteValue());
        }
    } else {
        for (int index = 0; index < searchKey.size(); ++index) {
            keyFieldSourceIndicator.add(0);
        }
    }
    // index key type information is not persisted, thus we extract type information from the record metadata
    if (searchKeyType.isEmpty()) {
        try {
            Dataset dSet = metadataNode.getDataset(jobId, dvName, dsName);
            String datatypeName = dSet.getItemTypeName();
            String datatypeDataverseName = dSet.getItemTypeDataverseName();
            ARecordType recordDt = (ARecordType) metadataNode.getDatatype(jobId, datatypeDataverseName, datatypeName).getDatatype();
            String metatypeName = dSet.getMetaItemTypeName();
            String metatypeDataverseName = dSet.getMetaItemTypeDataverseName();
            ARecordType metaDt = null;
            if (metatypeName != null && metatypeDataverseName != null) {
                metaDt = (ARecordType) metadataNode.getDatatype(jobId, metatypeDataverseName, metatypeName).getDatatype();
            }
            try {
                searchKeyType = KeyFieldTypeUtil.getKeyTypes(recordDt, metaDt, searchKey, keyFieldSourceIndicator);
            } catch (AlgebricksException e) {
                throw new MetadataException(e);
            }
        } catch (RemoteException re) {
            throw HyracksDataException.create(re);
        }
    }
    return new Index(dvName, dsName, indexName, indexStructure, searchKey, keyFieldSourceIndicator, searchKeyType, gramLength, isEnforcingKeys, isPrimaryIndex, pendingOp);
}

Example 4 with Index

use of org.apache.asterix.metadata.entities.Index in project asterixdb by apache.

the class ExternalIndexingOperations method buildAbortOp.

public static JobSpecification buildAbortOp(Dataset ds, List<Index> indexes, MetadataProvider metadataProvider) throws AlgebricksException {
    JobSpecification spec = RuntimeUtils.createJobSpecification(metadataProvider.getApplicationContext());
    IStorageManager storageMgr = metadataProvider.getStorageComponentProvider().getStorageManager();
    ArrayList<IIndexDataflowHelperFactory> treeDataflowHelperFactories = new ArrayList<>();
    AlgebricksPartitionConstraint constraints = null;
    for (Index index : indexes) {
        IFileSplitProvider indexSplitProvider;
        if (isValidIndexName(index.getDatasetName(), index.getIndexName())) {
            Pair<IFileSplitProvider, AlgebricksPartitionConstraint> sAndConstraints = metadataProvider.getSplitProviderAndConstraints(ds, index.getIndexName());
            indexSplitProvider = sAndConstraints.first;
            constraints = sAndConstraints.second;
        } else {
            indexSplitProvider = metadataProvider.getSplitProviderAndConstraints(ds, IndexingConstants.getFilesIndexName(ds.getDatasetName())).first;
        }
        IIndexDataflowHelperFactory indexDataflowHelperFactory = new IndexDataflowHelperFactory(storageMgr, indexSplitProvider);
        treeDataflowHelperFactories.add(indexDataflowHelperFactory);
    }
    ExternalDatasetIndexesAbortOperatorDescriptor op = new ExternalDatasetIndexesAbortOperatorDescriptor(spec, treeDataflowHelperFactories);
    spec.addRoot(op);
    AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, op, constraints);
    spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy());
    return spec;
}

Example 5 with Index

use of org.apache.asterix.metadata.entities.Index in project asterixdb by apache.

the class MetadataNode method addDataset.

@Override
public void addDataset(JobId jobId, Dataset dataset) throws MetadataException, RemoteException {
    try {
        // Insert into the 'dataset' dataset.
        DatasetTupleTranslator tupleReaderWriter = tupleTranslatorProvider.getDatasetTupleTranslator(true);
        ITupleReference datasetTuple = tupleReaderWriter.getTupleFromMetadataEntity(dataset);
        insertTupleIntoIndex(jobId, MetadataPrimaryIndexes.DATASET_DATASET, datasetTuple);
        if (dataset.getDatasetType() == DatasetType.INTERNAL) {
            // Add the primary index for the dataset.
            InternalDatasetDetails id = (InternalDatasetDetails) dataset.getDatasetDetails();
            Index primaryIndex = new Index(dataset.getDataverseName(), dataset.getDatasetName(), dataset.getDatasetName(), IndexType.BTREE, id.getPrimaryKey(), id.getKeySourceIndicator(), id.getPrimaryKeyType(), false, true, dataset.getPendingOp());
            addIndex(jobId, primaryIndex);
        }
    } catch (HyracksDataException e) {
        if (e.getComponent().equals(ErrorCode.HYRACKS) && e.getErrorCode() == ErrorCode.DUPLICATE_KEY) {
            throw new MetadataException("A dataset with this name " + dataset.getDatasetName() + " already exists in dataverse '" + dataset.getDataverseName() + "'.", e);
        } else {
            throw new MetadataException(e);
        }
    } catch (ACIDException e) {
        throw new MetadataException(e);
    }
}