Examples with DeepLabLabel org.vitrivr.cineast.core.features.neuralnet.tf.models.deeplab.DeepLabLabel used on opensource projects

Example 1 with DeepLabLabel

use of org.vitrivr.cineast.core.features.neuralnet.tf.models.deeplab.DeepLabLabel in project cineast by vitrivr.

the class ConceptMasksAde20k method processSegment.

@Override
public synchronized void processSegment(SegmentContainer shot) {
    if (shot == null || shot.getMostRepresentativeFrame() == null || shot.getMostRepresentativeFrame().getImage() == null || shot.getMostRepresentativeFrame().getImage() == VideoFrame.EMPTY_VIDEO_FRAME) {
        return;
    }
    if (this.ade20k == null) {
        try {
            this.ade20k = new DeepLabAde20k();
        } catch (RuntimeException e) {
            LOGGER.error(LogHelper.getStackTrace(e));
            return;
        }
    }
    TUint8 inputTensor = DeepLab.prepareImage(shot.getMostRepresentativeFrame().getImage().getBufferedImage());
    int[][] tmp = this.ade20k.processImage(inputTensor);
    List<DeepLabLabel> ade20kLabels = linearize(DeepLabLabel.fromAde20kId(tmp));
    inputTensor.close();
    ArrayList<LinkedList<DeepLabLabel>> ade20kPartitions = GridPartitioner.partition(ade20kLabels, tmp.length, tmp[0].length, GRID_PARTITIONS, GRID_PARTITIONS);
    float[] vector = new float[2 * GRID_PARTITIONS * GRID_PARTITIONS];
    for (int i = 0; i < GRID_PARTITIONS * GRID_PARTITIONS; ++i) {
        DeepLabLabel dominantLabel = DeepLabLabel.getDominantLabel(ade20kPartitions.get(i));
        vector[2 * i] = dominantLabel.getEmbeddX();
        vector[2 * i + 1] = dominantLabel.getEmbeddY();
    }
    persist(shot.getId(), new FloatVectorImpl(vector));
}

Example 2 with DeepLabLabel

use of org.vitrivr.cineast.core.features.neuralnet.tf.models.deeplab.DeepLabLabel in project cineast by vitrivr.

the class ConceptMasks method getSimilar.

@Override
public List<ScoreElement> getSimilar(SegmentContainer sc, ReadableQueryConfig qc) {
    Optional<SemanticMap> optional = sc.getSemanticMap();
    if (!optional.isPresent()) {
        return Collections.emptyList();
    }
    DeepLabLabel[][] labels = optional.get().getLabels();
    List<DeepLabLabel> list = linearize(labels);
    ArrayList<LinkedList<DeepLabLabel>> partitions = GridPartitioner.partition(list, labels.length, labels[0].length, GRID_PARTITIONS, GRID_PARTITIONS);
    float[] vector = new float[2 * GRID_PARTITIONS * GRID_PARTITIONS];
    float[] weights = new float[2 * GRID_PARTITIONS * GRID_PARTITIONS];
    for (int i = 0; i < GRID_PARTITIONS * GRID_PARTITIONS; ++i) {
        DeepLabLabel dominantLabel = DeepLabLabel.getDominantLabel(partitions.get(i));
        // TODO expose this to the API
        float weight = dominantLabel == DeepLabLabel.NOTHING ? 0f : 1f;
        weights[2 * i] = weight;
        weights[2 * i + 1] = weight;
        vector[2 * i] = dominantLabel.getEmbeddX();
        vector[2 * i + 1] = dominantLabel.getEmbeddY();
    }
    return this.getSimilar(vector, new QueryConfig(qc).setDistanceWeights(weights));
}

Example 3 with DeepLabLabel

use of org.vitrivr.cineast.core.features.neuralnet.tf.models.deeplab.DeepLabLabel in project cineast by vitrivr.

the class ConceptMasks method processSegment.

@Override
public synchronized void processSegment(SegmentContainer shot) {
    if (shot == null || shot.getMostRepresentativeFrame() == null || shot.getMostRepresentativeFrame().getImage() == null || shot.getMostRepresentativeFrame().getImage() == VideoFrame.EMPTY_VIDEO_FRAME) {
        return;
    }
    TUint8 inputTensor = DeepLab.prepareImage(shot.getMostRepresentativeFrame().getImage().getBufferedImage());
    int[][] tmp = this.ade20k.processImage(inputTensor);
    List<DeepLabLabel> ade20kLabels = linearize(DeepLabLabel.fromAde20kId(tmp));
    List<DeepLabLabel> cityscapesLabels = linearize(DeepLabLabel.fromCityscapesId(this.cityscapes.processImage(inputTensor)));
    List<DeepLabLabel> pascalvocLabels = linearize(DeepLabLabel.fromPascalVocId(this.pascalvoc.processImage(inputTensor)));
    inputTensor.close();
    ArrayList<LinkedList<DeepLabLabel>> ade20kPartitions = GridPartitioner.partition(ade20kLabels, tmp.length, tmp[0].length, GRID_PARTITIONS, GRID_PARTITIONS);
    ArrayList<LinkedList<DeepLabLabel>> cityscapesPartitions = GridPartitioner.partition(cityscapesLabels, tmp.length, tmp[0].length, GRID_PARTITIONS, GRID_PARTITIONS);
    ArrayList<LinkedList<DeepLabLabel>> pascalvocLabelsPartitions = GridPartitioner.partition(pascalvocLabels, tmp.length, tmp[0].length, GRID_PARTITIONS, GRID_PARTITIONS);
    float[] vector = new float[2 * GRID_PARTITIONS * GRID_PARTITIONS];
    for (int i = 0; i < GRID_PARTITIONS * GRID_PARTITIONS; ++i) {
        ArrayList<DeepLabLabel> labels = new ArrayList<>();
        labels.addAll(ade20kPartitions.get(i));
        labels.addAll(cityscapesPartitions.get(i));
        labels.addAll(pascalvocLabelsPartitions.get(i));
        DeepLabLabel dominantLabel = DeepLabLabel.getDominantLabel(labels);
        vector[2 * i] = dominantLabel.getEmbeddX();
        vector[2 * i + 1] = dominantLabel.getEmbeddY();
    }
    persist(shot.getId(), new FloatVectorImpl(vector));
}

Example 4 with DeepLabLabel

use of org.vitrivr.cineast.core.features.neuralnet.tf.models.deeplab.DeepLabLabel in project cineast by vitrivr.

the class ConceptMasksAde20k method getSimilar.

@Override
public List<ScoreElement> getSimilar(SegmentContainer sc, ReadableQueryConfig qc) {
    Optional<SemanticMap> optional = sc.getSemanticMap();
    if (!optional.isPresent()) {
        return Collections.emptyList();
    }
    DeepLabLabel[][] labels = optional.get().getLabels();
    List<DeepLabLabel> list = linearize(labels);
    ArrayList<LinkedList<DeepLabLabel>> ade20kPartitions = GridPartitioner.partition(list, labels.length, labels[0].length, GRID_PARTITIONS, GRID_PARTITIONS);
    float[] vector = new float[2 * GRID_PARTITIONS * GRID_PARTITIONS];
    float[] weights = new float[2 * GRID_PARTITIONS * GRID_PARTITIONS];
    for (int i = 0; i < GRID_PARTITIONS * GRID_PARTITIONS; ++i) {
        DeepLabLabel dominantLabel = DeepLabLabel.getDominantLabel(ade20kPartitions.get(i));
        // TODO expose this to the API
        float weight = dominantLabel == DeepLabLabel.NOTHING ? 0f : 1f;
        weights[2 * i] = weight;
        weights[2 * i + 1] = weight;
        vector[2 * i] = dominantLabel.getEmbeddX();
        vector[2 * i + 1] = dominantLabel.getEmbeddY();
    }
    return this.getSimilar(vector, new QueryConfig(qc).setDistanceWeights(weights));
}