Examples with HPCP org.vitrivr.cineast.core.util.audio.HPCP used on opensource projects

Example 1 with HPCP

use of org.vitrivr.cineast.core.util.audio.HPCP in project cineast by vitrivr.

the class HPCPShingle method getFeatures.

/**
 * Returns a list of feature vectors given a SegmentContainer.
 *
 * @param segment SegmentContainer for which to calculate the feature vectors.
 * @return List of HPCP Shingle feature vectors.
 */
private List<float[]> getFeatures(SegmentContainer segment) {
    /* Create STFT; If this fails, return empty list. */
    Pair<Integer, Integer> parameters = FFTUtil.parametersForDuration(segment.getSamplingrate(), WINDOW_SIZE);
    STFT stft = segment.getSTFT(parameters.first, (parameters.first - 2 * parameters.second) / 2, parameters.second, new HanningWindow());
    if (stft == null) {
        return new ArrayList<>(0);
    }
    HPCP hpcps = new HPCP(this.resolution, this.min_frequency, this.max_frequency);
    hpcps.addContribution(stft);
    int vectors = Math.max(hpcps.size() - SHINGLE_SIZE, 1);
    final SummaryStatistics statistics = new SummaryStatistics();
    List<Pair<Double, float[]>> features = new ArrayList<>(vectors);
    for (int n = 0; n < vectors; n++) {
        Pair<Double, float[]> feature = this.getHPCPShingle(hpcps, n);
        features.add(feature);
        statistics.addValue(feature.first);
    }
    final double threshold = 0.25 * statistics.getGeometricMean();
    return features.stream().filter(f -> (f.first > threshold)).map(f -> f.second).collect(Collectors.toList());
}

Example 2 with HPCP

use of org.vitrivr.cineast.core.util.audio.HPCP in project cineast by vitrivr.

the class CENS method preprocessQuery.

/**
 * This method represents the first step that's executed when processing query. The associated SegmentContainer is examined and feature-vectors are being generated. The generated vectors are returned by this method together with an optional weight-vector.
 *
 * <strong>Important: </strong> The weight-vector must have the same size as the feature-vectors returned by the method.
 *
 * @param sc SegmentContainer that was submitted to the feature module.
 * @param qc A QueryConfig object that contains query-related configuration parameters. Can still be edited.
 * @return List of feature vectors for lookup.
 */
@Override
protected List<float[]> preprocessQuery(SegmentContainer sc, ReadableQueryConfig qc) {
    /* Create STFT. If this fails, return empty list. */
    final Pair<Integer, Integer> parameters = FFTUtil.parametersForDuration(sc.getSamplingrate(), WINDOW_SIZE);
    final STFT stft = sc.getSTFT(parameters.first, (parameters.first - 2 * parameters.second) / 3, parameters.second, new BlackmanHarrisWindow());
    if (stft == null) {
        return new ArrayList<>(0);
    }
    /* Prepare empty features. */
    final List<float[]> features = new ArrayList<>(3 * QUERY_SETTINGS.length);
    /* Prepare HPCPs... */
    final HPCP hpcps = new HPCP(HPCP.Resolution.FULLSEMITONE, minFrequency, maxFrequency);
    hpcps.addContribution(stft);
    /*
     * ... and derive CENS features; only three features per QUERY_SETTING are kept because the shifts
     * resulting from the Shingeling are realised as individual vectors during ingest!
     */
    for (int[] QUERY_SETTING : QUERY_SETTINGS) {
        List<float[]> cens = this.getFeatures(hpcps, QUERY_SETTING[0], QUERY_SETTING[1]);
        for (int i = 0; i < cens.size(); i++) {
            if (i % SHINGLE_SIZE == 0) {
                features.add(cens.get(i));
            }
        }
    }
    return features;
}

Example 3 with HPCP

use of org.vitrivr.cineast.core.util.audio.HPCP in project cineast by vitrivr.

the class AverageHPCP method getFeatures.

/**
 * Returns a list of feature vectors given a SegmentContainer.
 *
 * @param segment SegmentContainer for which to calculate the feature vectors.
 * @return List of HPCP Shingle feature vectors.
 */
private List<float[]> getFeatures(SegmentContainer segment) {
    /* Create STFT. IF this fails, return empty list. */
    Pair<Integer, Integer> parameters = FFTUtil.parametersForDuration(segment.getSamplingrate(), WINDOW_SIZE);
    STFT stft = segment.getSTFT(parameters.first, (parameters.first - 2 * parameters.second) / 2, parameters.second, new HanningWindow());
    if (stft == null) {
        return new ArrayList<>();
    }
    HPCP hpcps = new HPCP(this.resolution, this.min_frequency, this.max_frequency);
    hpcps.addContribution(stft);
    /* Determine number of vectors that will result from the data. */
    int vectors = hpcps.size() / this.average;
    List<float[]> features = new ArrayList<>(vectors);
    for (int i = 0; i < vectors; i++) {
        float[] feature = new float[2 * this.resolution.bins];
        SummaryStatistics[] statistics = new SummaryStatistics[this.resolution.bins];
        for (int j = 0; j < this.average; j++) {
            for (int k = 0; k < this.resolution.bins; k++) {
                if (statistics[k] == null) {
                    statistics[k] = new SummaryStatistics();
                }
                statistics[k].addValue(hpcps.getHpcp(i * this.average + j)[k]);
            }
        }
        for (int k = 0; k < this.resolution.bins; k++) {
            feature[2 * k] = (float) statistics[k].getMean();
            feature[2 * k + 1] = (float) statistics[k].getStandardDeviation();
        }
        features.add(MathHelper.normalizeL2(feature));
    }
    return features;
}

Example 4 with HPCP

use of org.vitrivr.cineast.core.util.audio.HPCP in project cineast by vitrivr.

the class CENS method processSegment.

/**
 * Processes a SegmentContainer for later persisting it in the storage layer.
 *
 * @param sc The SegmentContainer that should be processed.
 */
@Override
public void processSegment(SegmentContainer sc) {
    /* Create STFT. If this fails, return empty list. */
    Pair<Integer, Integer> parameters = FFTUtil.parametersForDuration(sc.getSamplingrate(), WINDOW_SIZE);
    STFT stft = sc.getSTFT(parameters.first, (parameters.first - 2 * parameters.second) / 3, parameters.second, new HanningWindow());
    if (stft == null) {
        return;
    }
    /* Prepare HPCPs... */
    HPCP hpcps = new HPCP(HPCP.Resolution.FULLSEMITONE, minFrequency, maxFrequency);
    hpcps.addContribution(stft);
    List<float[]> features = this.getFeatures(hpcps, 41, 10);
    features.forEach(f -> this.persist(sc.getId(), new FloatVectorImpl(f)));
}

Example 5 with HPCP

use of org.vitrivr.cineast.core.util.audio.HPCP in project cineast by vitrivr.

the class CENSExporter method processSegment.

@Override
public void processSegment(SegmentContainer shot) {
    /* IF shot has no samples, this step is skipped. */
    if (shot.getNumberOfSamples() == 0) {
        return;
    }
    /* Prepare STFT and HPCP for the segment. */
    final Path directory = this.destination.resolve(shot.getSuperId());
    final STFT stft = shot.getSTFT(2048, 512, new HanningWindow());
    final HPCP hpcp = new HPCP();
    hpcp.addContribution(stft);
    double[][] cens = CENS.cens(hpcp, 25, 5);
    /* Visualize chromagram and write it to disc. */
    try {
        BufferedImage image = AudioSignalVisualizer.visualizeCens(cens, this.width, this.height);
        if (image != null) {
            Files.createDirectories(directory);
            ImageIO.write(image, "JPEG", directory.resolve(shot.getId() + ".jpg").toFile());
        } else {
            LOGGER.warn("CENS chromagram could not be visualized!");
        }
    } catch (IOException exception) {
        LOGGER.error("A serious error occurred while writing the CENS chromagram image! ({})", LogHelper.getStackTrace(exception));
    }
}