Examples with IBinaryComparatorFactory org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory used on opensource projects

Example 1 with IBinaryComparatorFactory

use of org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory in project asterixdb by apache.

the class RangeMapBuilder method verifyRangeOrder.

public static void verifyRangeOrder(IRangeMap rangeMap, boolean ascending) throws CompilationException {
    // TODO Add support for composite fields.
    int fieldIndex = 0;
    int fieldType = rangeMap.getTag(0, 0);
    BinaryComparatorFactoryProvider comparatorFactory = BinaryComparatorFactoryProvider.INSTANCE;
    IBinaryComparatorFactory bcf = comparatorFactory.getBinaryComparatorFactory(ATypeTag.VALUE_TYPE_MAPPING[fieldType], ascending);
    IBinaryComparator comparator = bcf.createBinaryComparator();
    int c = 0;
    for (int split = 1; split < rangeMap.getSplitCount(); ++split) {
        if (fieldType != rangeMap.getTag(fieldIndex, split)) {
            throw new CompilationException("Range field contains more than a single type of items (" + fieldType + " and " + rangeMap.getTag(fieldIndex, split) + ").");
        }
        int previousSplit = split - 1;
        try {
            c = comparator.compare(rangeMap.getByteArray(fieldIndex, previousSplit), rangeMap.getStartOffset(fieldIndex, previousSplit), rangeMap.getLength(fieldIndex, previousSplit), rangeMap.getByteArray(fieldIndex, split), rangeMap.getStartOffset(fieldIndex, split), rangeMap.getLength(fieldIndex, split));
        } catch (HyracksDataException e) {
            throw new CompilationException(e);
        }
        if (c >= 0) {
            throw new CompilationException("Range fields are not in sorted order.");
        }
    }
}

Example 2 with IBinaryComparatorFactory

use of org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory in project asterixdb by apache.

the class MetadataProvider method getComparatorFactoriesAndTypeTraitsOfSecondaryBTreeIndex.

private Pair<IBinaryComparatorFactory[], ITypeTraits[]> getComparatorFactoriesAndTypeTraitsOfSecondaryBTreeIndex(List<List<String>> sidxKeyFieldNames, List<IAType> sidxKeyFieldTypes, List<List<String>> pidxKeyFieldNames, ARecordType recType, DatasetType dsType, boolean hasMeta, List<Integer> primaryIndexKeyIndicators, List<Integer> secondaryIndexIndicators, ARecordType metaType) throws AlgebricksException {
    IBinaryComparatorFactory[] comparatorFactories;
    ITypeTraits[] typeTraits;
    int sidxKeyFieldCount = sidxKeyFieldNames.size();
    int pidxKeyFieldCount = pidxKeyFieldNames.size();
    typeTraits = new ITypeTraits[sidxKeyFieldCount + pidxKeyFieldCount];
    comparatorFactories = new IBinaryComparatorFactory[sidxKeyFieldCount + pidxKeyFieldCount];
    int i = 0;
    for (; i < sidxKeyFieldCount; ++i) {
        Pair<IAType, Boolean> keyPairType = Index.getNonNullableOpenFieldType(sidxKeyFieldTypes.get(i), sidxKeyFieldNames.get(i), (hasMeta && secondaryIndexIndicators.get(i).intValue() == 1) ? metaType : recType);
        IAType keyType = keyPairType.first;
        comparatorFactories[i] = BinaryComparatorFactoryProvider.INSTANCE.getBinaryComparatorFactory(keyType, true);
        typeTraits[i] = TypeTraitProvider.INSTANCE.getTypeTrait(keyType);
    }
    for (int j = 0; j < pidxKeyFieldCount; ++j, ++i) {
        IAType keyType = null;
        try {
            switch(dsType) {
                case INTERNAL:
                    keyType = (hasMeta && primaryIndexKeyIndicators.get(j).intValue() == 1) ? metaType.getSubFieldType(pidxKeyFieldNames.get(j)) : recType.getSubFieldType(pidxKeyFieldNames.get(j));
                    break;
                case EXTERNAL:
                    keyType = IndexingConstants.getFieldType(j);
                    break;
                default:
                    throw new AlgebricksException("Unknown Dataset Type");
            }
        } catch (AsterixException e) {
            throw new AlgebricksException(e);
        }
        comparatorFactories[i] = BinaryComparatorFactoryProvider.INSTANCE.getBinaryComparatorFactory(keyType, true);
        typeTraits[i] = TypeTraitProvider.INSTANCE.getTypeTrait(keyType);
    }
    return new Pair<>(comparatorFactories, typeTraits);
}

Example 3 with IBinaryComparatorFactory

use of org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory in project asterixdb by apache.

the class BTreeResourceFactoryProvider method getResourceFactory.

@Override
public IResourceFactory getResourceFactory(MetadataProvider mdProvider, Dataset dataset, Index index, ARecordType recordType, ARecordType metaType, ILSMMergePolicyFactory mergePolicyFactory, Map<String, String> mergePolicyProperties, ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories) throws AlgebricksException {
    int[] filterFields = IndexUtil.getFilterFields(dataset, index, filterTypeTraits);
    int[] btreeFields = IndexUtil.getBtreeFieldsIfFiltered(dataset, index);
    IStorageComponentProvider storageComponentProvider = mdProvider.getStorageComponentProvider();
    ITypeTraits[] typeTraits = getTypeTraits(mdProvider, dataset, index, recordType, metaType);
    IBinaryComparatorFactory[] cmpFactories = getCmpFactories(mdProvider, dataset, index, recordType, metaType);
    int[] bloomFilterFields = getBloomFilterFields(dataset, index);
    boolean durable = !dataset.isTemp();
    double bloomFilterFalsePositiveRate = mdProvider.getStorageProperties().getBloomFilterFalsePositiveRate();
    ILSMOperationTrackerFactory opTrackerFactory = dataset.getIndexOperationTrackerFactory(index);
    ILSMIOOperationCallbackFactory ioOpCallbackFactory = dataset.getIoOperationCallbackFactory(index);
    IStorageManager storageManager = storageComponentProvider.getStorageManager();
    IMetadataPageManagerFactory metadataPageManagerFactory = storageComponentProvider.getMetadataPageManagerFactory();
    ILSMIOOperationSchedulerProvider ioSchedulerProvider = storageComponentProvider.getIoOperationSchedulerProvider();
    switch(dataset.getDatasetType()) {
        case EXTERNAL:
            return index.getIndexName().equals(IndexingConstants.getFilesIndexName(dataset.getDatasetName())) ? new ExternalBTreeLocalResourceFactory(storageManager, typeTraits, cmpFactories, filterTypeTraits, filterCmpFactories, filterFields, opTrackerFactory, ioOpCallbackFactory, metadataPageManagerFactory, ioSchedulerProvider, mergePolicyFactory, mergePolicyProperties, durable, bloomFilterFields, bloomFilterFalsePositiveRate, false, btreeFields) : new ExternalBTreeWithBuddyLocalResourceFactory(storageManager, typeTraits, cmpFactories, filterTypeTraits, filterCmpFactories, filterFields, opTrackerFactory, ioOpCallbackFactory, metadataPageManagerFactory, ioSchedulerProvider, mergePolicyFactory, mergePolicyProperties, durable, bloomFilterFields, bloomFilterFalsePositiveRate, false, btreeFields);
        case INTERNAL:
            AsterixVirtualBufferCacheProvider vbcProvider = new AsterixVirtualBufferCacheProvider(dataset.getDatasetId());
            return new LSMBTreeLocalResourceFactory(storageManager, typeTraits, cmpFactories, filterTypeTraits, filterCmpFactories, filterFields, opTrackerFactory, ioOpCallbackFactory, metadataPageManagerFactory, vbcProvider, ioSchedulerProvider, mergePolicyFactory, mergePolicyProperties, durable, bloomFilterFields, bloomFilterFalsePositiveRate, index.isPrimaryIndex(), btreeFields);
        default:
            throw new CompilationException(ErrorCode.COMPILATION_UNKNOWN_DATASET_TYPE, dataset.getDatasetType().toString());
    }
}

Example 4 with IBinaryComparatorFactory

use of org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory in project asterixdb by apache.

the class BTreeResourceFactoryProvider method getCmpFactories.

private static IBinaryComparatorFactory[] getCmpFactories(MetadataProvider metadataProvider, Dataset dataset, Index index, ARecordType recordType, ARecordType metaType) throws AlgebricksException {
    IBinaryComparatorFactory[] primaryCmpFactories = dataset.getPrimaryComparatorFactories(metadataProvider, recordType, metaType);
    if (index.isPrimaryIndex()) {
        return dataset.getPrimaryComparatorFactories(metadataProvider, recordType, metaType);
    } else if (dataset.getDatasetType() == DatasetType.EXTERNAL && index.getIndexName().equals(IndexingConstants.getFilesIndexName(dataset.getDatasetName()))) {
        return FilesIndexDescription.FILES_INDEX_COMP_FACTORIES;
    }
    int numPrimaryKeys = dataset.getPrimaryKeys().size();
    int numSecondaryKeys = index.getKeyFieldNames().size();
    IBinaryComparatorFactoryProvider cmpFactoryProvider = metadataProvider.getStorageComponentProvider().getComparatorFactoryProvider();
    IBinaryComparatorFactory[] secondaryCmpFactories = new IBinaryComparatorFactory[numSecondaryKeys + numPrimaryKeys];
    for (int i = 0; i < numSecondaryKeys; i++) {
        ARecordType sourceType;
        List<Integer> keySourceIndicators = index.getKeyFieldSourceIndicators();
        if (keySourceIndicators == null || keySourceIndicators.get(i) == 0) {
            sourceType = recordType;
        } else {
            sourceType = metaType;
        }
        Pair<IAType, Boolean> keyTypePair = Index.getNonNullableOpenFieldType(index.getKeyFieldTypes().get(i), index.getKeyFieldNames().get(i), sourceType);
        IAType keyType = keyTypePair.first;
        secondaryCmpFactories[i] = cmpFactoryProvider.getBinaryComparatorFactory(keyType, true);
    }
    // Add serializers and comparators for primary index fields.
    for (int i = 0; i < numPrimaryKeys; i++) {
        secondaryCmpFactories[numSecondaryKeys + i] = primaryCmpFactories[i];
    }
    return secondaryCmpFactories;
}

Example 5 with IBinaryComparatorFactory

use of org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory in project asterixdb by apache.

the class MetadataBootstrap method enlistMetadataDataset.

/**
     * Enlist a metadata index so it is available for metadata operations
     * should be performed upon bootstrapping
     *
     * @param index
     * @throws HyracksDataException
     */
public static void enlistMetadataDataset(INCServiceContext ncServiceCtx, IMetadataIndex index) throws HyracksDataException {
    ClusterPartition metadataPartition = appContext.getMetadataProperties().getMetadataPartition();
    int metadataDeviceId = metadataPartition.getIODeviceNum();
    String metadataPartitionPath = StoragePathUtil.prepareStoragePartitionPath(ClusterProperties.INSTANCE.getStorageDirectoryName(), metadataPartition.getPartitionId());
    String resourceName = metadataPartitionPath + File.separator + index.getFileNameRelativePath();
    FileReference file = ioManager.getFileReference(metadataDeviceId, resourceName);
    index.setFile(file);
    ITypeTraits[] typeTraits = index.getTypeTraits();
    IBinaryComparatorFactory[] cmpFactories = index.getKeyBinaryComparatorFactory();
    int[] bloomFilterKeyFields = index.getBloomFilterKeyFields();
    // opTrackerProvider and ioOpCallbackFactory should both be acquired through IStorageManager
    // We are unable to do this since IStorageManager needs a dataset to determine the appropriate
    // objects
    ILSMOperationTrackerFactory opTrackerFactory = index.isPrimaryIndex() ? new PrimaryIndexOperationTrackerFactory(index.getDatasetId().getId()) : new SecondaryIndexOperationTrackerFactory(index.getDatasetId().getId());
    ILSMIOOperationCallbackFactory ioOpCallbackFactory = LSMBTreeIOOperationCallbackFactory.INSTANCE;
    IStorageComponentProvider storageComponentProvider = appContext.getStorageComponentProvider();
    if (isNewUniverse()) {
        LSMBTreeLocalResourceFactory lsmBtreeFactory = new LSMBTreeLocalResourceFactory(storageComponentProvider.getStorageManager(), typeTraits, cmpFactories, null, null, null, opTrackerFactory, ioOpCallbackFactory, storageComponentProvider.getMetadataPageManagerFactory(), new AsterixVirtualBufferCacheProvider(index.getDatasetId().getId()), storageComponentProvider.getIoOperationSchedulerProvider(), appContext.getMetadataMergePolicyFactory(), GlobalConfig.DEFAULT_COMPACTION_POLICY_PROPERTIES, true, bloomFilterKeyFields, appContext.getBloomFilterFalsePositiveRate(), true, null);
        DatasetLocalResourceFactory dsLocalResourceFactory = new DatasetLocalResourceFactory(index.getDatasetId().getId(), lsmBtreeFactory);
        // TODO(amoudi) Creating the index should be done through the same code path as other indexes
        // This is to be done by having a metadata dataset associated with each index
        IIndexBuilder indexBuilder = new IndexBuilder(ncServiceCtx, storageComponentProvider.getStorageManager(), index::getResourceId, file, dsLocalResourceFactory, true);
        indexBuilder.build();
    } else {
        final LocalResource resource = localResourceRepository.get(file.getRelativePath());
        if (resource == null) {
            throw new HyracksDataException("Could not find required metadata indexes. Please delete " + appContext.getMetadataProperties().getTransactionLogDirs().get(appContext.getTransactionSubsystem().getId()) + " to intialize as a new instance. (WARNING: all data will be lost.)");
        }
        // Why do we care about metadata dataset's resource ids? why not assign them ids similar to other resources?
        if (index.getResourceId() != resource.getId()) {
            throw new HyracksDataException("Resource Id doesn't match expected metadata index resource id");
        }
        IndexDataflowHelper indexHelper = new IndexDataflowHelper(ncServiceCtx, storageComponentProvider.getStorageManager(), file);
        // Opening the index through the helper will ensure it gets instantiated
        indexHelper.open();
        indexHelper.close();
    }
}