Examples with TensorType com.yahoo.tensor.TensorType used on opensource projects

Example 16 with TensorType

use of com.yahoo.tensor.TensorType in project vespa by vespa-engine.

the class Concat method concatenateTo.

private void concatenateTo(IndexedTensor a, IndexedTensor b, long offset, TensorType concatType, int[] aToIndexes, int[] bToIndexes, Tensor.Builder builder) {
    Set<String> otherADimensions = a.type().dimensionNames().stream().filter(d -> !d.equals(dimension)).collect(Collectors.toSet());
    for (Iterator<IndexedTensor.SubspaceIterator> ia = a.subspaceIterator(otherADimensions); ia.hasNext(); ) {
        IndexedTensor.SubspaceIterator iaSubspace = ia.next();
        TensorAddress aAddress = iaSubspace.address();
        for (Iterator<IndexedTensor.SubspaceIterator> ib = b.subspaceIterator(otherADimensions); ib.hasNext(); ) {
            IndexedTensor.SubspaceIterator ibSubspace = ib.next();
            while (ibSubspace.hasNext()) {
                Tensor.Cell bCell = ibSubspace.next();
                TensorAddress combinedAddress = combineAddresses(aAddress, aToIndexes, bCell.getKey(), bToIndexes, concatType, offset, dimension);
                // incompatible
                if (combinedAddress == null)
                    continue;
                builder.cell(combinedAddress, bCell.getValue());
            }
            iaSubspace.reset();
        }
    }
}

Example 17 with TensorType

use of com.yahoo.tensor.TensorType in project vespa by vespa-engine.

the class MixedBinaryFormat method decodeCells.

private void decodeCells(GrowableByteBuffer buffer, MixedTensor.BoundBuilder builder, TensorType type) {
    List<TensorType.Dimension> sparseDimensions = type.dimensions().stream().filter(d -> !d.isIndexed()).collect(Collectors.toList());
    TensorType sparseType = MixedTensor.createPartialType(sparseDimensions);
    long denseSubspaceSize = builder.denseSubspaceSize();
    int numBlocks = 1;
    if (sparseDimensions.size() > 0) {
        numBlocks = buffer.getInt1_4Bytes();
    }
    double[] denseSubspace = new double[(int) denseSubspaceSize];
    for (int i = 0; i < numBlocks; ++i) {
        TensorAddress.Builder sparseAddress = new TensorAddress.Builder(sparseType);
        for (TensorType.Dimension sparseDimension : sparseDimensions) {
            sparseAddress.add(sparseDimension.name(), buffer.getUtf8String());
        }
        for (long denseOffset = 0; denseOffset < denseSubspaceSize; denseOffset++) {
            denseSubspace[(int) denseOffset] = buffer.getDouble();
        }
        builder.block(sparseAddress.build(), denseSubspace);
    }
}

Example 18 with TensorType

use of com.yahoo.tensor.TensorType in project vespa by vespa-engine.

the class MixedBinaryFormat method decode.

@Override
public Tensor decode(Optional<TensorType> optionalType, GrowableByteBuffer buffer) {
    TensorType type;
    if (optionalType.isPresent()) {
        type = optionalType.get();
        TensorType serializedType = decodeType(buffer);
        if (!serializedType.isAssignableTo(type))
            throw new IllegalArgumentException("Type/instance mismatch: A tensor of type " + serializedType + " cannot be assigned to type " + type);
    } else {
        type = decodeType(buffer);
    }
    MixedTensor.BoundBuilder builder = (MixedTensor.BoundBuilder) MixedTensor.Builder.of(type);
    decodeCells(buffer, builder, type);
    return builder.build();
}

Example 19 with TensorType

use of com.yahoo.tensor.TensorType in project vespa by vespa-engine.

the class MapEvaluationTypeContext method tensorFeatureType.

/**
 * There are two features which returns the (non-empty) tensor type: tensorFromLabels and tensorFromWeightedSet.
 * This returns the type of those features if this is a reference to either of them, or empty otherwise.
 */
private Optional<TensorType> tensorFeatureType(Reference reference) {
    if (!reference.name().equals("tensorFromLabels") && !reference.name().equals("tensorFromWeightedSet"))
        return Optional.empty();
    if (reference.arguments().size() != 1 && reference.arguments().size() != 2)
        throw new IllegalArgumentException(reference.name() + " must have one or two arguments");
    ExpressionNode arg0 = reference.arguments().expressions().get(0);
    if (!(arg0 instanceof ReferenceNode) || !FeatureNames.isSimpleFeature(((ReferenceNode) arg0).reference()))
        throw new IllegalArgumentException("The first argument of " + reference.name() + " must be a simple feature, not " + arg0);
    String dimension;
    if (reference.arguments().size() > 1) {
        ExpressionNode arg1 = reference.arguments().expressions().get(1);
        if ((!(arg1 instanceof ReferenceNode) || !(((ReferenceNode) arg1).reference().isIdentifier())) && (!(arg1 instanceof NameNode)))
            throw new IllegalArgumentException("The second argument of " + reference.name() + " must be a dimension name, not " + arg1);
        dimension = reference.arguments().expressions().get(1).toString();
    } else {
        // default
        dimension = ((ReferenceNode) arg0).reference().arguments().expressions().get(0).toString();
    }
    return Optional.of(new TensorType.Builder().mapped(dimension).build());
}

Example 20 with TensorType

use of com.yahoo.tensor.TensorType in project vespa by vespa-engine.

the class MapEvaluationTypeContext method getType.

@Override
public TensorType getType(Reference reference) {
    // A reference to a macro argument?
    Optional<String> binding = boundIdentifier(reference);
    if (binding.isPresent()) {
        try {
            // than their string values requires deeper changes
            return new RankingExpression(binding.get()).type(this);
        } catch (ParseException e) {
            throw new IllegalArgumentException(e);
        }
    }
    // A reference to an attribute, query or constant feature?
    if (FeatureNames.isSimpleFeature(reference)) {
        // The argument may be a local identifier bound to the actual value
        String argument = reference.simpleArgument().get();
        reference = Reference.simple(reference.name(), bindings.getOrDefault(argument, argument));
        return featureTypes.getOrDefault(reference, defaultTypeOf(reference));
    }
    // A reference to a function?
    Optional<ExpressionFunction> function = functionInvocation(reference);
    if (function.isPresent()) {
        return function.get().getBody().type(this.withBindings(bind(function.get().arguments(), reference.arguments())));
    }
    // A reference to a feature which returns a tensor?
    Optional<TensorType> featureTensorType = tensorFeatureType(reference);
    if (featureTensorType.isPresent()) {
        return featureTensorType.get();
    }
    // all match features
    return TensorType.empty;
}