Examples with IBinaryHashFunctionFactory org.apache.hyracks.api.dataflow.value.IBinaryHashFunctionFactory used on opensource projects

Example 16 with IBinaryHashFunctionFactory

use of org.apache.hyracks.api.dataflow.value.IBinaryHashFunctionFactory in project asterixdb by apache.

the class JoinMultiComparator method contributeRuntimeOperator.

@Override
public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException {
    int[] keysLeft = JobGenHelper.variablesToFieldIndexes(keysLeftBranch, inputSchemas[0]);
    int[] keysRight = JobGenHelper.variablesToFieldIndexes(keysRightBranch, inputSchemas[1]);
    IVariableTypeEnvironment env = context.getTypeEnvironment(op);
    IBinaryHashFunctionFactory[] hashFunFactories = JobGenHelper.variablesToBinaryHashFunctionFactories(keysLeftBranch, env, context);
    IBinaryHashFunctionFamily[] hashFunFamilies = JobGenHelper.variablesToBinaryHashFunctionFamilies(keysLeftBranch, env, context);
    IBinaryComparatorFactory[] comparatorFactories = new IBinaryComparatorFactory[keysLeft.length];
    int i = 0;
    IBinaryComparatorFactoryProvider bcfp = context.getBinaryComparatorFactoryProvider();
    for (LogicalVariable v : keysLeftBranch) {
        Object t = env.getVarType(v);
        comparatorFactories[i++] = bcfp.getBinaryComparatorFactory(t, true);
    }
    IPredicateEvaluatorFactoryProvider predEvaluatorFactoryProvider = context.getPredicateEvaluatorFactoryProvider();
    IPredicateEvaluatorFactory predEvaluatorFactory = predEvaluatorFactoryProvider == null ? null : predEvaluatorFactoryProvider.getPredicateEvaluatorFactory(keysLeft, keysRight);
    RecordDescriptor recDescriptor = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), propagatedSchema, context);
    IOperatorDescriptorRegistry spec = builder.getJobSpec();
    IOperatorDescriptor opDesc;
    boolean optimizedHashJoin = true;
    for (IBinaryHashFunctionFamily family : hashFunFamilies) {
        if (family == null) {
            optimizedHashJoin = false;
            break;
        }
    }
    if (optimizedHashJoin) {
        opDesc = generateOptimizedHashJoinRuntime(context, inputSchemas, keysLeft, keysRight, hashFunFamilies, comparatorFactories, predEvaluatorFactory, recDescriptor, spec);
    } else {
        opDesc = generateHashJoinRuntime(context, inputSchemas, keysLeft, keysRight, hashFunFactories, comparatorFactories, predEvaluatorFactory, recDescriptor, spec);
    }
    contributeOpDesc(builder, (AbstractLogicalOperator) op, opDesc);
    ILogicalOperator src1 = op.getInputs().get(0).getValue();
    builder.contributeGraphEdge(src1, 0, op, 0);
    ILogicalOperator src2 = op.getInputs().get(1).getValue();
    builder.contributeGraphEdge(src2, 0, op, 1);
}

Example 17 with IBinaryHashFunctionFactory

use of org.apache.hyracks.api.dataflow.value.IBinaryHashFunctionFactory in project asterixdb by apache.

the class SortMergeExchangePOperator method createConnectorDescriptor.

@Override
public Pair<IConnectorDescriptor, TargetConstraint> createConnectorDescriptor(IConnectorDescriptorRegistry spec, ILogicalOperator op, IOperatorSchema opSchema, JobGenContext context) throws AlgebricksException {
    int n = sortColumns.length;
    int[] sortFields = new int[n];
    IBinaryComparatorFactory[] comps = new IBinaryComparatorFactory[n];
    IBinaryHashFunctionFactory[] hashFuns = new IBinaryHashFunctionFactory[n];
    IVariableTypeEnvironment env = context.getTypeEnvironment(op);
    INormalizedKeyComputerFactoryProvider nkcfProvider = context.getNormalizedKeyComputerFactoryProvider();
    INormalizedKeyComputerFactory nkcf = null;
    for (int i = 0; i < n; i++) {
        sortFields[i] = opSchema.findVariable(sortColumns[i].getColumn());
        Object type = env.getVarType(sortColumns[i].getColumn());
        IBinaryComparatorFactoryProvider bcfp = context.getBinaryComparatorFactoryProvider();
        comps[i] = bcfp.getBinaryComparatorFactory(type, sortColumns[i].getOrder() == OrderKind.ASC);
        IBinaryHashFunctionFactoryProvider bhffp = context.getBinaryHashFunctionFactoryProvider();
        hashFuns[i] = bhffp.getBinaryHashFunctionFactory(type);
        if (i == 0 && nkcfProvider != null && type != null) {
            nkcf = nkcfProvider.getNormalizedKeyComputerFactory(type, sortColumns[i].getOrder() == OrderKind.ASC);
        }
    }
    ITuplePartitionComputerFactory tpcf = new FieldHashPartitionComputerFactory(sortFields, hashFuns);
    IConnectorDescriptor conn = new MToNPartitioningMergingConnectorDescriptor(spec, tpcf, sortFields, comps, nkcf);
    return new Pair<IConnectorDescriptor, TargetConstraint>(conn, TargetConstraint.ONE);
}

Example 18 with IBinaryHashFunctionFactory

use of org.apache.hyracks.api.dataflow.value.IBinaryHashFunctionFactory in project asterixdb by apache.

the class JobGenHelper method variablesToBinaryHashFunctionFactories.

public static IBinaryHashFunctionFactory[] variablesToBinaryHashFunctionFactories(Collection<LogicalVariable> varLogical, IVariableTypeEnvironment env, JobGenContext context) throws AlgebricksException {
    IBinaryHashFunctionFactory[] funFactories = new IBinaryHashFunctionFactory[varLogical.size()];
    int i = 0;
    IBinaryHashFunctionFactoryProvider bhffProvider = context.getBinaryHashFunctionFactoryProvider();
    for (LogicalVariable var : varLogical) {
        Object type = env.getVarType(var);
        funFactories[i++] = bhffProvider.getBinaryHashFunctionFactory(type);
    }
    return funFactories;
}

Example 19 with IBinaryHashFunctionFactory

use of org.apache.hyracks.api.dataflow.value.IBinaryHashFunctionFactory in project asterixdb by apache.

the class Dataset method getPrimaryHashFunctionFactories.

/**
     * Gets the hash function factories for the primary key fields of this dataset.
     *
     * @param metadataProvider,
     *            the metadata provider.
     * @return the hash function factories for the primary key fields of this dataset.
     * @throws AlgebricksException
     */
public IBinaryHashFunctionFactory[] getPrimaryHashFunctionFactories(MetadataProvider metadataProvider) throws AlgebricksException {
    ARecordType recordType = (ARecordType) metadataProvider.findType(this);
    ARecordType metaType = (ARecordType) metadataProvider.findMetaType(this);
    List<List<String>> partitioningKeys = getPrimaryKeys();
    int numPrimaryKeys = partitioningKeys.size();
    IBinaryHashFunctionFactory[] hashFuncFactories = new IBinaryHashFunctionFactory[numPrimaryKeys];
    List<Integer> indicators = null;
    if (hasMetaPart()) {
        indicators = ((InternalDatasetDetails) getDatasetDetails()).getKeySourceIndicator();
    }
    for (int i = 0; i < numPrimaryKeys; i++) {
        IAType keyType = (indicators == null || indicators.get(i) == 0) ? recordType.getSubFieldType(partitioningKeys.get(i)) : metaType.getSubFieldType(partitioningKeys.get(i));
        hashFuncFactories[i] = BinaryHashFunctionFactoryProvider.INSTANCE.getBinaryHashFunctionFactory(keyType);
    }
    return hashFuncFactories;
}

Example 20 with IBinaryHashFunctionFactory

use of org.apache.hyracks.api.dataflow.value.IBinaryHashFunctionFactory in project asterixdb by apache.

the class ScanPrintTest method scanPrint02.

@Test
public void scanPrint02() throws Exception {
    JobSpecification spec = new JobSpecification();
    FileSplit[] ordersSplits = new FileSplit[] { new ManagedFileSplit(NC2_ID, "data" + File.separator + "tpch0.001" + File.separator + "orders.tbl") };
    IFileSplitProvider ordersSplitsProvider = new ConstantFileSplitProvider(ordersSplits);
    RecordDescriptor ordersDesc = new RecordDescriptor(new ISerializerDeserializer[] { new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer() });
    FileScanOperatorDescriptor ordScanner = new FileScanOperatorDescriptor(spec, ordersSplitsProvider, new DelimitedDataTupleParserFactory(new IValueParserFactory[] { UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE }, '|'), ordersDesc);
    PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, ordScanner, NC2_ID);
    ResultSetId rsId = new ResultSetId(1);
    IOperatorDescriptor printer = new ResultWriterOperatorDescriptor(spec, rsId, true, false, ResultSerializerFactoryProvider.INSTANCE.getResultSerializerFactoryProvider());
    spec.addResultSetId(rsId);
    PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer, NC2_ID);
    IConnectorDescriptor conn1 = new MToNPartitioningConnectorDescriptor(spec, new FieldHashPartitionComputerFactory(new int[] { 0 }, new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory.of(UTF8StringPointable.FACTORY) }));
    spec.connect(conn1, ordScanner, 0, printer, 0);
    spec.addRoot(printer);
    runTest(spec);
}