Examples with STFT org.vitrivr.cineast.core.util.dsp.fft.STFT used on opensource projects

Example 1 with STFT

use of org.vitrivr.cineast.core.util.dsp.fft.STFT in project cineast by vitrivr.

the class MFCCShingle method getFeatures.

/**
 * Derives and returns a list of MFCC features for a SegmentContainer.
 *
 * @param segment SegmentContainer to derive the MFCC features from.
 * @return List of MFCC Shingles.
 */
private List<float[]> getFeatures(SegmentContainer segment) {
    final Pair<Integer, Integer> parameters = FFTUtil.parametersForDuration(segment.getSamplingrate(), WINDOW_SIZE);
    final STFT stft = segment.getSTFT(parameters.first, (parameters.first - 2 * parameters.second) / 2, parameters.second, new HanningWindow());
    if (stft == null) {
        return new ArrayList<>(0);
    }
    final List<MFCC> mfccs = MFCC.calculate(stft);
    int vectors = mfccs.size() - SHINGLE_SIZE;
    List<float[]> features = new ArrayList<>(Math.max(1, vectors));
    if (vectors > 0) {
        for (int i = 0; i < vectors; i++) {
            float[] feature = new float[SHINGLE_SIZE * 13];
            for (int j = 0; j < SHINGLE_SIZE; j++) {
                MFCC mfcc = mfccs.get(i + j);
                System.arraycopy(mfcc.getCepstra(), 0, feature, 13 * j, 13);
            }
            if (MathHelper.checkNotZero(feature) && MathHelper.checkNotNaN(feature)) {
                features.add(MathHelper.normalizeL2(feature));
            }
        }
    }
    return features;
}

Example 2 with STFT

use of org.vitrivr.cineast.core.util.dsp.fft.STFT in project cineast by vitrivr.

the class HPCPShingle method getFeatures.

/**
 * Returns a list of feature vectors given a SegmentContainer.
 *
 * @param segment SegmentContainer for which to calculate the feature vectors.
 * @return List of HPCP Shingle feature vectors.
 */
private List<float[]> getFeatures(SegmentContainer segment) {
    /* Create STFT; If this fails, return empty list. */
    Pair<Integer, Integer> parameters = FFTUtil.parametersForDuration(segment.getSamplingrate(), WINDOW_SIZE);
    STFT stft = segment.getSTFT(parameters.first, (parameters.first - 2 * parameters.second) / 2, parameters.second, new HanningWindow());
    if (stft == null) {
        return new ArrayList<>(0);
    }
    HPCP hpcps = new HPCP(this.resolution, this.min_frequency, this.max_frequency);
    hpcps.addContribution(stft);
    int vectors = Math.max(hpcps.size() - SHINGLE_SIZE, 1);
    final SummaryStatistics statistics = new SummaryStatistics();
    List<Pair<Double, float[]>> features = new ArrayList<>(vectors);
    for (int n = 0; n < vectors; n++) {
        Pair<Double, float[]> feature = this.getHPCPShingle(hpcps, n);
        features.add(feature);
        statistics.addValue(feature.first);
    }
    final double threshold = 0.25 * statistics.getGeometricMean();
    return features.stream().filter(f -> (f.first > threshold)).map(f -> f.second).collect(Collectors.toList());
}

Example 3 with STFT

use of org.vitrivr.cineast.core.util.dsp.fft.STFT in project cineast by vitrivr.

the class CENS method preprocessQuery.

/**
 * This method represents the first step that's executed when processing query. The associated SegmentContainer is examined and feature-vectors are being generated. The generated vectors are returned by this method together with an optional weight-vector.
 *
 * <strong>Important: </strong> The weight-vector must have the same size as the feature-vectors returned by the method.
 *
 * @param sc SegmentContainer that was submitted to the feature module.
 * @param qc A QueryConfig object that contains query-related configuration parameters. Can still be edited.
 * @return List of feature vectors for lookup.
 */
@Override
protected List<float[]> preprocessQuery(SegmentContainer sc, ReadableQueryConfig qc) {
    /* Create STFT. If this fails, return empty list. */
    final Pair<Integer, Integer> parameters = FFTUtil.parametersForDuration(sc.getSamplingrate(), WINDOW_SIZE);
    final STFT stft = sc.getSTFT(parameters.first, (parameters.first - 2 * parameters.second) / 3, parameters.second, new BlackmanHarrisWindow());
    if (stft == null) {
        return new ArrayList<>(0);
    }
    /* Prepare empty features. */
    final List<float[]> features = new ArrayList<>(3 * QUERY_SETTINGS.length);
    /* Prepare HPCPs... */
    final HPCP hpcps = new HPCP(HPCP.Resolution.FULLSEMITONE, minFrequency, maxFrequency);
    hpcps.addContribution(stft);
    /*
     * ... and derive CENS features; only three features per QUERY_SETTING are kept because the shifts
     * resulting from the Shingeling are realised as individual vectors during ingest!
     */
    for (int[] QUERY_SETTING : QUERY_SETTINGS) {
        List<float[]> cens = this.getFeatures(hpcps, QUERY_SETTING[0], QUERY_SETTING[1]);
        for (int i = 0; i < cens.size(); i++) {
            if (i % SHINGLE_SIZE == 0) {
                features.add(cens.get(i));
            }
        }
    }
    return features;
}

Example 4 with STFT

use of org.vitrivr.cineast.core.util.dsp.fft.STFT in project cineast by vitrivr.

the class AudioSTFTProvider method getSTFT.

/**
 * Calculates and returns the Short-term Fourier Transform of the current AudioSegment.
 *
 * @param windowsize Size of the window used during STFT. Must be a power of two.
 * @param overlap    Overlap in samples between two subsequent windows.
 * @param function   WindowFunction to apply before calculating the STFT.
 * @return STFT of the current AudioSegment.
 */
default STFT getSTFT(int windowsize, int overlap, int padding, WindowFunction function) {
    double[] data = new double[windowsize];
    Arrays.fill(data, 0.0);
    STFT stft = new STFT(windowsize, overlap, padding, function, 22050);
    stft.forward(data);
    return stft;
}

Example 5 with STFT

use of org.vitrivr.cineast.core.util.dsp.fft.STFT in project cineast by vitrivr.

the class AudioSegment method getSTFT.

/**
 * Calculates and returns the Short-term Fourier Transform of the current AudioSegment.
 *
 * @param windowsize Size of the window used during STFT. Must be a power of two.
 * @param overlap    Overlap in samples between two subsequent windows.
 * @param padding    Zero-padding before and after the actual sample data. Causes the window to contain (windowsize-2*padding) data-points..
 * @param function   WindowFunction to apply before calculating the STFT.
 * @return STFT of the current AudioSegment or null if the segment is empty.
 */
@Override
public STFT getSTFT(int windowsize, int overlap, int padding, WindowFunction function) {
    if (2 * padding >= windowsize) {
        throw new IllegalArgumentException("The combined padding must be smaller than the sample window.");
    }
    STFT stft = new STFT(windowsize, overlap, padding, function, this.descriptor.getSamplingrate());
    stft.forward(this.getMeanSamplesAsDouble());
    return stft;
}