Examples with ITypeTraits org.apache.hyracks.api.dataflow.value.ITypeTraits used on opensource projects

Example 1 with ITypeTraits

use of org.apache.hyracks.api.dataflow.value.ITypeTraits in project asterixdb by apache.

the class RTreeResourceFactoryProvider method getTypeTraits.

private static ITypeTraits[] getTypeTraits(MetadataProvider metadataProvider, Dataset dataset, Index index, ARecordType recordType, ARecordType metaType) throws AlgebricksException {
    ITypeTraitProvider ttProvider = metadataProvider.getStorageComponentProvider().getTypeTraitProvider();
    List<List<String>> secondaryKeyFields = index.getKeyFieldNames();
    int numSecondaryKeys = secondaryKeyFields.size();
    int numPrimaryKeys = dataset.getPrimaryKeys().size();
    ITypeTraits[] primaryTypeTraits = dataset.getPrimaryTypeTraits(metadataProvider, recordType, metaType);
    if (numSecondaryKeys != 1) {
        throw new AsterixException("Cannot use " + numSecondaryKeys + " fields as a key for the R-tree index. " + "There can be only one field as a key for the R-tree index.");
    }
    ARecordType sourceType;
    List<Integer> keySourceIndicators = index.getKeyFieldSourceIndicators();
    if (keySourceIndicators == null || keySourceIndicators.get(0) == 0) {
        sourceType = recordType;
    } else {
        sourceType = metaType;
    }
    Pair<IAType, Boolean> spatialTypePair = Index.getNonNullableOpenFieldType(index.getKeyFieldTypes().get(0), secondaryKeyFields.get(0), sourceType);
    IAType spatialType = spatialTypePair.first;
    if (spatialType == null) {
        throw new AsterixException("Could not find field " + secondaryKeyFields.get(0) + " in the schema.");
    }
    int numDimensions = NonTaggedFormatUtil.getNumDimensions(spatialType.getTypeTag());
    int numNestedSecondaryKeyFields = numDimensions * 2;
    ITypeTraits[] secondaryTypeTraits = new ITypeTraits[numNestedSecondaryKeyFields + numPrimaryKeys];
    IAType nestedKeyType = NonTaggedFormatUtil.getNestedSpatialType(spatialType.getTypeTag());
    for (int i = 0; i < numNestedSecondaryKeyFields; i++) {
        secondaryTypeTraits[i] = ttProvider.getTypeTrait(nestedKeyType);
    }
    for (int i = 0; i < numPrimaryKeys; i++) {
        secondaryTypeTraits[numNestedSecondaryKeyFields + i] = primaryTypeTraits[i];
    }
    return secondaryTypeTraits;
}

Example 2 with ITypeTraits

use of org.apache.hyracks.api.dataflow.value.ITypeTraits in project asterixdb by apache.

the class SecondaryIndexOperationsHelper method createExternalIndexingOp.

// This method creates a source indexing operator for external data
protected ExternalScanOperatorDescriptor createExternalIndexingOp(JobSpecification spec) throws AlgebricksException {
    // A record + primary keys
    ISerializerDeserializer[] serdes = new ISerializerDeserializer[1 + numPrimaryKeys];
    ITypeTraits[] typeTraits = new ITypeTraits[1 + numPrimaryKeys];
    // payload serde and type traits for the record slot
    serdes[0] = payloadSerde;
    typeTraits[0] = TypeTraitProvider.INSTANCE.getTypeTrait(itemType);
    //  serdes and type traits for rid fields
    for (int i = 1; i < serdes.length; i++) {
        serdes[i] = IndexingConstants.getSerializerDeserializer(i - 1);
        typeTraits[i] = IndexingConstants.getTypeTraits(i - 1);
    }
    // output record desc
    RecordDescriptor indexerDesc = new RecordDescriptor(serdes, typeTraits);
    // Create the operator and its partition constraits
    Pair<ExternalScanOperatorDescriptor, AlgebricksPartitionConstraint> indexingOpAndConstraints;
    try {
        indexingOpAndConstraints = ExternalIndexingOperations.createExternalIndexingOp(spec, metadataProvider, dataset, itemType, indexerDesc, externalFiles);
    } catch (Exception e) {
        throw new AlgebricksException(e);
    }
    AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, indexingOpAndConstraints.first, indexingOpAndConstraints.second);
    // Set the primary partition constraints to this partition constraints
    primaryPartitionConstraint = indexingOpAndConstraints.second;
    return indexingOpAndConstraints.first;
}

Example 3 with ITypeTraits

use of org.apache.hyracks.api.dataflow.value.ITypeTraits in project asterixdb by apache.

the class AbstractRTreeOperatorTest method setup.

@Before
public void setup() throws Exception {
    testHelper = createTestHelper();
    primaryFileName = testHelper.getPrimaryIndexName();
    primarySplitProvider = new ConstantFileSplitProvider(new FileSplit[] { new ManagedFileSplit(NC1_ID, primaryFileName) });
    primaryHelperFactory = new IndexDataflowHelperFactory(storageManager, primarySplitProvider);
    secondaryFileName = testHelper.getSecondaryIndexName();
    secondarySplitProvider = new ConstantFileSplitProvider(new FileSplit[] { new ManagedFileSplit(NC1_ID, secondaryFileName) });
    secondaryHelperFactory = new IndexDataflowHelperFactory(storageManager, secondarySplitProvider);
    // field, type and key declarations for primary index
    primaryTypeTraits[0] = UTF8StringPointable.TYPE_TRAITS;
    primaryTypeTraits[1] = UTF8StringPointable.TYPE_TRAITS;
    primaryTypeTraits[2] = UTF8StringPointable.TYPE_TRAITS;
    primaryTypeTraits[3] = UTF8StringPointable.TYPE_TRAITS;
    primaryTypeTraits[4] = UTF8StringPointable.TYPE_TRAITS;
    primaryTypeTraits[5] = UTF8StringPointable.TYPE_TRAITS;
    primaryTypeTraits[6] = UTF8StringPointable.TYPE_TRAITS;
    primaryTypeTraits[7] = UTF8StringPointable.TYPE_TRAITS;
    primaryTypeTraits[8] = UTF8StringPointable.TYPE_TRAITS;
    primaryTypeTraits[9] = UTF8StringPointable.TYPE_TRAITS;
    primaryComparatorFactories[0] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY);
    // field, type and key declarations for secondary indexes
    secondaryTypeTraits[0] = DoublePointable.TYPE_TRAITS;
    secondaryTypeTraits[1] = DoublePointable.TYPE_TRAITS;
    secondaryTypeTraits[2] = DoublePointable.TYPE_TRAITS;
    secondaryTypeTraits[3] = DoublePointable.TYPE_TRAITS;
    secondaryTypeTraits[4] = UTF8StringPointable.TYPE_TRAITS;
    secondaryComparatorFactories[0] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY);
    secondaryComparatorFactories[1] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY);
    secondaryComparatorFactories[2] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY);
    secondaryComparatorFactories[3] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY);
    // This only used for LSMRTree
    int[] rtreeFields = null;
    int[] filterFields = null;
    ITypeTraits[] filterTypes = null;
    IBinaryComparatorFactory[] filterCmpFactories = null;
    if (rTreeType == RTreeType.LSMRTREE || rTreeType == RTreeType.LSMRTREE_WITH_ANTIMATTER) {
        rtreeFields = new int[] { 0, 1, 2, 3, 4 };
        filterFields = new int[] { 4 };
        filterTypes = new ITypeTraits[] { UTF8StringPointable.TYPE_TRAITS };
        filterCmpFactories = new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY) };
    }
    int[] btreeFields = null;
    if (rTreeType == RTreeType.LSMRTREE) {
        btreeKeyFieldCount = 1;
        btreeComparatorFactories = new IBinaryComparatorFactory[btreeKeyFieldCount];
        btreeComparatorFactories[0] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY);
        btreeFields = new int[] { 4 };
    } else {
        btreeComparatorFactories[0] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY);
        btreeComparatorFactories[1] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY);
        btreeComparatorFactories[2] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY);
        btreeComparatorFactories[3] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY);
        btreeComparatorFactories[4] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY);
    }
    IPrimitiveValueProviderFactory[] secondaryValueProviderFactories = RTreeUtils.createPrimitiveValueProviderFactories(secondaryComparatorFactories.length, DoublePointable.FACTORY);
    rtreeFactory = createSecondaryResourceFactory(secondaryValueProviderFactories, RTreePolicyType.RSTARTREE, btreeComparatorFactories, LSMRTreeUtils.proposeBestLinearizer(secondaryTypeTraits, secondaryComparatorFactories.length), btreeFields);
}

Example 4 with ITypeTraits

use of org.apache.hyracks.api.dataflow.value.ITypeTraits in project asterixdb by apache.

the class OrderedIndexExamplesTest method bulkLoadExample.

/**
     * Bulk load example. Load a tree with 100,000 tuples. BTree has a composite
     * key to "simulate" non-unique index creation.
     */
@Test
public void bulkLoadExample() throws Exception {
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info("Bulk load example");
    }
    // Declare fields.
    int fieldCount = 3;
    ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
    typeTraits[0] = IntegerPointable.TYPE_TRAITS;
    typeTraits[1] = IntegerPointable.TYPE_TRAITS;
    typeTraits[2] = IntegerPointable.TYPE_TRAITS;
    // Declare field serdes.
    ISerializerDeserializer[] fieldSerdes = { IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE };
    // declare keys
    int keyFieldCount = 2;
    IBinaryComparatorFactory[] cmpFactories = new IBinaryComparatorFactory[keyFieldCount];
    cmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
    cmpFactories[1] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
    // This is only used for the LSM-BTree.
    int[] bloomFilterKeyFields = new int[keyFieldCount];
    bloomFilterKeyFields[0] = 0;
    bloomFilterKeyFields[1] = 1;
    ITreeIndex treeIndex = createTreeIndex(typeTraits, cmpFactories, bloomFilterKeyFields, null, null, null, null);
    treeIndex.create();
    treeIndex.activate();
    // Load sorted records.
    int ins = 100000;
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info("Bulk loading " + ins + " tuples");
    }
    long start = System.currentTimeMillis();
    IIndexBulkLoader bulkLoader = treeIndex.createBulkLoader(0.7f, false, ins, true);
    ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
    ArrayTupleReference tuple = new ArrayTupleReference();
    for (int i = 0; i < ins; i++) {
        TupleUtils.createIntegerTuple(tb, tuple, i, i, 5);
        bulkLoader.add(tuple);
    }
    bulkLoader.end();
    long end = System.currentTimeMillis();
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info(ins + " tuples loaded in " + (end - start) + "ms");
    }
    IIndexAccessor indexAccessor = treeIndex.createAccessor(TestOperationCallback.INSTANCE, TestOperationCallback.INSTANCE);
    // Build low key.
    ArrayTupleBuilder lowKeyTb = new ArrayTupleBuilder(1);
    ArrayTupleReference lowKey = new ArrayTupleReference();
    TupleUtils.createIntegerTuple(lowKeyTb, lowKey, 44444);
    // Build high key.
    ArrayTupleBuilder highKeyTb = new ArrayTupleBuilder(1);
    ArrayTupleReference highKey = new ArrayTupleReference();
    TupleUtils.createIntegerTuple(highKeyTb, highKey, 44500);
    // Prefix-Range search in [44444, 44500]
    rangeSearch(cmpFactories, indexAccessor, fieldSerdes, lowKey, highKey, null, null);
    treeIndex.validate();
    treeIndex.deactivate();
    treeIndex.destroy();
}

Example 5 with ITypeTraits

use of org.apache.hyracks.api.dataflow.value.ITypeTraits in project asterixdb by apache.

the class OrderedIndexExamplesTest method deleteExample.

/**
     * Deletion Example. Create a BTree with one variable-length key field and
     * one variable-length value field. Fill B-tree with random values using
     * insertions, then delete entries one-by-one. Repeat procedure a few times
     * on same BTree.
     */
@Test
public void deleteExample() throws Exception {
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info("Deletion Example");
    }
    // Declare fields.
    int fieldCount = 2;
    ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
    typeTraits[0] = UTF8StringPointable.TYPE_TRAITS;
    typeTraits[1] = UTF8StringPointable.TYPE_TRAITS;
    // Declare field serdes.
    ISerializerDeserializer[] fieldSerdes = { new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer() };
    // Declare keys.
    int keyFieldCount = 1;
    IBinaryComparatorFactory[] cmpFactories = new IBinaryComparatorFactory[keyFieldCount];
    cmpFactories[0] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY);
    // This is only used for the LSM-BTree.
    int[] bloomFilterKeyFields = new int[keyFieldCount];
    bloomFilterKeyFields[0] = 0;
    ITreeIndex treeIndex = createTreeIndex(typeTraits, cmpFactories, bloomFilterKeyFields, null, null, null, null);
    treeIndex.create();
    treeIndex.activate();
    ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
    ArrayTupleReference tuple = new ArrayTupleReference();
    IIndexAccessor indexAccessor = treeIndex.createAccessor(TestOperationCallback.INSTANCE, TestOperationCallback.INSTANCE);
    // Max string length to be generated.
    int runs = 3;
    for (int run = 0; run < runs; run++) {
        if (LOGGER.isLoggable(Level.INFO)) {
            LOGGER.info("Deletion example run: " + (run + 1) + "/" + runs);
            LOGGER.info("Inserting into tree...");
        }
        int maxLength = 10;
        int ins = 10000;
        String[] f0s = new String[ins];
        String[] f1s = new String[ins];
        int insDone = 0;
        int[] insDoneCmp = new int[ins];
        for (int i = 0; i < ins; i++) {
            String f0 = randomString(Math.abs(rnd.nextInt()) % maxLength + 1, rnd);
            String f1 = randomString(Math.abs(rnd.nextInt()) % maxLength + 1, rnd);
            TupleUtils.createTuple(tb, tuple, fieldSerdes, f0, f1);
            f0s[i] = f0;
            f1s[i] = f1;
            if (LOGGER.isLoggable(Level.INFO)) {
                if (i % 1000 == 0) {
                    LOGGER.info("Inserting " + i);
                }
            }
            try {
                indexAccessor.insert(tuple);
                insDone++;
            } catch (HyracksDataException e) {
                if (e.getErrorCode() != ErrorCode.DUPLICATE_KEY) {
                    throw e;
                }
            }
            insDoneCmp[i] = insDone;
        }
        if (LOGGER.isLoggable(Level.INFO)) {
            LOGGER.info("Deleting from tree...");
        }
        int delDone = 0;
        for (int i = 0; i < ins; i++) {
            TupleUtils.createTuple(tb, tuple, fieldSerdes, f0s[i], f1s[i]);
            if (LOGGER.isLoggable(Level.INFO)) {
                if (i % 1000 == 0) {
                    LOGGER.info("Deleting " + i);
                }
            }
            try {
                indexAccessor.delete(tuple);
                delDone++;
            } catch (HyracksDataException e) {
                if (e.getErrorCode() != ErrorCode.UPDATE_OR_DELETE_NON_EXISTENT_KEY) {
                    throw e;
                }
            }
            if (insDoneCmp[i] != delDone) {
                if (LOGGER.isLoggable(Level.INFO)) {
                    LOGGER.info("INCONSISTENT STATE, ERROR IN DELETION EXAMPLE.");
                    LOGGER.info("INSDONECMP: " + insDoneCmp[i] + " " + delDone);
                }
                break;
            }
        }
        if (insDone != delDone) {
            if (LOGGER.isLoggable(Level.INFO)) {
                LOGGER.info("ERROR! INSDONE: " + insDone + " DELDONE: " + delDone);
            }
            break;
        }
    }
    treeIndex.validate();
    treeIndex.deactivate();
    treeIndex.destroy();
}