Examples with IndexType org.apache.asterix.common.config.DatasetConfig.IndexType used on opensource projects

Example 6 with IndexType

use of org.apache.asterix.common.config.DatasetConfig.IndexType in project asterixdb by apache.

the class SecondaryInvertedIndexOperationsHelper method setSecondaryRecDescAndComparators.

@Override
@SuppressWarnings("rawtypes")
protected void setSecondaryRecDescAndComparators() throws AlgebricksException {
    int numSecondaryKeys = index.getKeyFieldNames().size();
    IndexType indexType = index.getIndexType();
    boolean isEnforcingKeyTypes = index.isEnforcingKeyFileds();
    // Sanity checks.
    if (numPrimaryKeys > 1) {
        throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_INDEX_FOR_DATASET_WITH_COMPOSITE_PRIMARY_INDEX, indexType, RecordUtil.toFullyQualifiedName(dataset.getDataverseName(), dataset.getDatasetName()));
    }
    if (numSecondaryKeys > 1) {
        throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_INDEX_NUM_OF_FIELD, numSecondaryKeys, indexType, 1);
    }
    if (indexType == IndexType.LENGTH_PARTITIONED_WORD_INVIX || indexType == IndexType.LENGTH_PARTITIONED_NGRAM_INVIX) {
        isPartitioned = true;
    } else {
        isPartitioned = false;
    }
    // Prepare record descriptor used in the assign op, and the optional
    // select op.
    secondaryFieldAccessEvalFactories = new IScalarEvaluatorFactory[numSecondaryKeys + numFilterFields];
    ISerializerDeserializer[] secondaryRecFields = new ISerializerDeserializer[numPrimaryKeys + numSecondaryKeys + numFilterFields];
    ISerializerDeserializer[] enforcedRecFields = new ISerializerDeserializer[1 + numPrimaryKeys + numFilterFields];
    secondaryTypeTraits = new ITypeTraits[numSecondaryKeys + numPrimaryKeys];
    ITypeTraits[] enforcedTypeTraits = new ITypeTraits[1 + numPrimaryKeys];
    ISerializerDeserializerProvider serdeProvider = FormatUtils.getDefaultFormat().getSerdeProvider();
    ITypeTraitProvider typeTraitProvider = FormatUtils.getDefaultFormat().getTypeTraitProvider();
    if (numSecondaryKeys > 0) {
        secondaryFieldAccessEvalFactories[0] = FormatUtils.getDefaultFormat().getFieldAccessEvaluatorFactory(isEnforcingKeyTypes ? enforcedItemType : itemType, index.getKeyFieldNames().get(0), numPrimaryKeys);
        Pair<IAType, Boolean> keyTypePair = Index.getNonNullableOpenFieldType(index.getKeyFieldTypes().get(0), index.getKeyFieldNames().get(0), itemType);
        secondaryKeyType = keyTypePair.first;
        anySecondaryKeyIsNullable = anySecondaryKeyIsNullable || keyTypePair.second;
        ISerializerDeserializer keySerde = serdeProvider.getSerializerDeserializer(secondaryKeyType);
        secondaryRecFields[0] = keySerde;
        secondaryTypeTraits[0] = typeTraitProvider.getTypeTrait(secondaryKeyType);
    }
    if (numFilterFields > 0) {
        secondaryFieldAccessEvalFactories[numSecondaryKeys] = FormatUtils.getDefaultFormat().getFieldAccessEvaluatorFactory(itemType, filterFieldName, numPrimaryKeys);
        Pair<IAType, Boolean> keyTypePair = Index.getNonNullableKeyFieldType(filterFieldName, itemType);
        IAType type = keyTypePair.first;
        ISerializerDeserializer serde = serdeProvider.getSerializerDeserializer(type);
        secondaryRecFields[numPrimaryKeys + numSecondaryKeys] = serde;
    }
    secondaryRecDesc = new RecordDescriptor(secondaryRecFields);
    // Comparators and type traits for tokens.
    int numTokenFields = (!isPartitioned) ? numSecondaryKeys : numSecondaryKeys + 1;
    tokenComparatorFactories = new IBinaryComparatorFactory[numTokenFields];
    tokenTypeTraits = new ITypeTraits[numTokenFields];
    tokenComparatorFactories[0] = NonTaggedFormatUtil.getTokenBinaryComparatorFactory(secondaryKeyType);
    tokenTypeTraits[0] = NonTaggedFormatUtil.getTokenTypeTrait(secondaryKeyType);
    if (isPartitioned) {
        // The partitioning field is hardcoded to be a short *without* an Asterix type tag.
        tokenComparatorFactories[1] = PointableBinaryComparatorFactory.of(ShortPointable.FACTORY);
        tokenTypeTraits[1] = ShortPointable.TYPE_TRAITS;
    }
    // Set tokenizer factory.
    // TODO: We might want to expose the hashing option at the AQL level,
    // and add the choice to the index metadata.
    tokenizerFactory = NonTaggedFormatUtil.getBinaryTokenizerFactory(secondaryKeyType.getTypeTag(), indexType, index.getGramLength());
    // Type traits for inverted-list elements. Inverted lists contain
    // primary keys.
    invListsTypeTraits = new ITypeTraits[numPrimaryKeys];
    if (numPrimaryKeys > 0) {
        invListsTypeTraits[0] = primaryRecDesc.getTypeTraits()[0];
        enforcedRecFields[0] = primaryRecDesc.getFields()[0];
        enforcedTypeTraits[0] = primaryRecDesc.getTypeTraits()[0];
    }
    enforcedRecFields[numPrimaryKeys] = serdeProvider.getSerializerDeserializer(itemType);
    enforcedRecDesc = new RecordDescriptor(enforcedRecFields, enforcedTypeTraits);
    // For tokenization, sorting and loading.
    // One token (+ optional partitioning field) + primary keys.
    numTokenKeyPairFields = (!isPartitioned) ? 1 + numPrimaryKeys : 2 + numPrimaryKeys;
    ISerializerDeserializer[] tokenKeyPairFields = new ISerializerDeserializer[numTokenKeyPairFields + numFilterFields];
    ITypeTraits[] tokenKeyPairTypeTraits = new ITypeTraits[numTokenKeyPairFields];
    tokenKeyPairComparatorFactories = new IBinaryComparatorFactory[numTokenKeyPairFields];
    tokenKeyPairFields[0] = serdeProvider.getSerializerDeserializer(secondaryKeyType);
    tokenKeyPairTypeTraits[0] = tokenTypeTraits[0];
    tokenKeyPairComparatorFactories[0] = NonTaggedFormatUtil.getTokenBinaryComparatorFactory(secondaryKeyType);
    int pkOff = 1;
    if (isPartitioned) {
        tokenKeyPairFields[1] = ShortSerializerDeserializer.INSTANCE;
        tokenKeyPairTypeTraits[1] = tokenTypeTraits[1];
        tokenKeyPairComparatorFactories[1] = PointableBinaryComparatorFactory.of(ShortPointable.FACTORY);
        pkOff = 2;
    }
    if (numPrimaryKeys > 0) {
        tokenKeyPairFields[pkOff] = primaryRecDesc.getFields()[0];
        tokenKeyPairTypeTraits[pkOff] = primaryRecDesc.getTypeTraits()[0];
        tokenKeyPairComparatorFactories[pkOff] = primaryComparatorFactories[0];
    }
    if (numFilterFields > 0) {
        tokenKeyPairFields[numPrimaryKeys + pkOff] = secondaryRecFields[numPrimaryKeys + numSecondaryKeys];
    }
    tokenKeyPairRecDesc = new RecordDescriptor(tokenKeyPairFields, tokenKeyPairTypeTraits);
    if (filterFieldName != null) {
        invertedIndexFields = new int[numTokenKeyPairFields];
        for (int i = 0; i < invertedIndexFields.length; i++) {
            invertedIndexFields[i] = i;
        }
        secondaryFilterFieldsForNonBulkLoadOps = new int[numFilterFields];
        secondaryFilterFieldsForNonBulkLoadOps[0] = numSecondaryKeys + numPrimaryKeys;
        invertedIndexFieldsForNonBulkLoadOps = new int[numSecondaryKeys + numPrimaryKeys];
        for (int i = 0; i < invertedIndexFieldsForNonBulkLoadOps.length; i++) {
            invertedIndexFieldsForNonBulkLoadOps[i] = i;
        }
    }
}

Example 7 with IndexType

use of org.apache.asterix.common.config.DatasetConfig.IndexType in project asterixdb by apache.

the class InvertedIndexResourceFactoryProvider method getTokenComparatorFactories.

private static IBinaryComparatorFactory[] getTokenComparatorFactories(Dataset dataset, Index index, ARecordType recordType, ARecordType metaType) throws AlgebricksException {
    int numPrimaryKeys = dataset.getPrimaryKeys().size();
    int numSecondaryKeys = index.getKeyFieldNames().size();
    IndexType indexType = index.getIndexType();
    // Sanity checks.
    if (numPrimaryKeys > 1) {
        throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_INDEX_FOR_DATASET_WITH_COMPOSITE_PRIMARY_INDEX, indexType, RecordUtil.toFullyQualifiedName(dataset.getDataverseName(), dataset.getDatasetName()));
    }
    if (numSecondaryKeys > 1) {
        throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_INDEX_NUM_OF_FIELD, numSecondaryKeys, indexType, 1);
    }
    boolean isPartitioned = indexType == IndexType.LENGTH_PARTITIONED_WORD_INVIX || indexType == IndexType.LENGTH_PARTITIONED_NGRAM_INVIX;
    List<Integer> keySourceIndicators = index.getKeyFieldSourceIndicators();
    ARecordType sourceType;
    if (keySourceIndicators == null || keySourceIndicators.get(0) == 0) {
        sourceType = recordType;
    } else {
        sourceType = metaType;
    }
    Pair<IAType, Boolean> keyTypePair = Index.getNonNullableOpenFieldType(index.getKeyFieldTypes().get(0), index.getKeyFieldNames().get(0), sourceType);
    IAType secondaryKeyType = keyTypePair.first;
    // Comparators and type traits for tokens.
    int numTokenFields = (!isPartitioned) ? numSecondaryKeys : numSecondaryKeys + 1;
    IBinaryComparatorFactory[] tokenComparatorFactories = new IBinaryComparatorFactory[numTokenFields];
    tokenComparatorFactories[0] = NonTaggedFormatUtil.getTokenBinaryComparatorFactory(secondaryKeyType);
    if (isPartitioned) {
        // The partitioning field is hardcoded to be a short *without* an Asterix type tag.
        tokenComparatorFactories[1] = PointableBinaryComparatorFactory.of(ShortPointable.FACTORY);
    }
    return tokenComparatorFactories;
}