Search in sources :

Example 1 with Variable

use of org.flyte.api.v1.Variable in project tika by apache.

the class NetCDFParser method parse.

/*
     * (non-Javadoc)
     * 
     * @see org.apache.tika.parser.Parser#parse(java.io.InputStream,
     * org.xml.sax.ContentHandler, org.apache.tika.metadata.Metadata,
     * org.apache.tika.parser.ParseContext)
     */
public void parse(InputStream stream, ContentHandler handler, Metadata metadata, ParseContext context) throws IOException, SAXException, TikaException {
    TemporaryResources tmp = TikaInputStream.isTikaInputStream(stream) ? null : new TemporaryResources();
    TikaInputStream tis = TikaInputStream.get(stream, tmp);
    NetcdfFile ncFile = null;
    try {
        ncFile = NetcdfFile.open(tis.getFile().getAbsolutePath());
        metadata.set("File-Type-Description", ncFile.getFileTypeDescription());
        // first parse out the set of global attributes
        for (Attribute attr : ncFile.getGlobalAttributes()) {
            Property property = resolveMetadataKey(attr.getFullName());
            if (attr.getDataType().isString()) {
                metadata.add(property, attr.getStringValue());
            } else if (attr.getDataType().isNumeric()) {
                int value = attr.getNumericValue().intValue();
                metadata.add(property, String.valueOf(value));
            }
        }
        XHTMLContentHandler xhtml = new XHTMLContentHandler(handler, metadata);
        xhtml.startDocument();
        xhtml.newline();
        xhtml.element("h1", "dimensions");
        xhtml.startElement("ul");
        xhtml.newline();
        for (Dimension dim : ncFile.getDimensions()) {
            xhtml.element("li", dim.getFullName() + " = " + dim.getLength());
        }
        xhtml.endElement("ul");
        xhtml.element("h1", "variables");
        xhtml.startElement("ul");
        xhtml.newline();
        for (Variable var : ncFile.getVariables()) {
            xhtml.startElement("li");
            xhtml.characters(var.getDataType() + " " + var.getNameAndDimensions());
            xhtml.newline();
            List<Attribute> attributes = var.getAttributes();
            if (!attributes.isEmpty()) {
                xhtml.startElement("ul");
                for (Attribute element : attributes) {
                    xhtml.element("li", element.toString());
                }
                xhtml.endElement("ul");
            }
            xhtml.endElement("li");
        }
        xhtml.endElement("ul");
        xhtml.endDocument();
    } catch (IOException e) {
        throw new TikaException("NetCDF parse error", e);
    } finally {
        if (ncFile != null) {
            ncFile.close();
        }
        if (tmp != null) {
            tmp.dispose();
        }
    }
}
Also used : NetcdfFile(ucar.nc2.NetcdfFile) Variable(ucar.nc2.Variable) TikaException(org.apache.tika.exception.TikaException) Attribute(ucar.nc2.Attribute) TemporaryResources(org.apache.tika.io.TemporaryResources) TikaInputStream(org.apache.tika.io.TikaInputStream) Dimension(ucar.nc2.Dimension) IOException(java.io.IOException) XHTMLContentHandler(org.apache.tika.sax.XHTMLContentHandler) Property(org.apache.tika.metadata.Property)

Example 2 with Variable

use of org.flyte.api.v1.Variable in project mzmine2 by mzmine.

the class NetCDFReadTask method startReading.

public void startReading() throws IOException {
    // Open NetCDF-file
    try {
        inputFile = NetcdfFile.open(file.getPath());
    } catch (Exception e) {
        logger.severe(e.toString());
        throw (new IOException("Couldn't open input file" + file));
    }
    /*
     * DEBUG: dump all variables for (Variable v : inputFile.getVariables()) {
     * System.out.println("variable " + v.getShortName()); }
     */
    // Find mass_values and intensity_values variables
    massValueVariable = inputFile.findVariable("mass_values");
    if (massValueVariable == null) {
        logger.severe("Could not find variable mass_values");
        throw (new IOException("Could not find variable mass_values"));
    }
    assert (massValueVariable.getRank() == 1);
    Attribute massScaleFacAttr = massValueVariable.findAttribute("scale_factor");
    if (massScaleFacAttr != null) {
        massValueScaleFactor = massScaleFacAttr.getNumericValue().doubleValue();
    }
    intensityValueVariable = inputFile.findVariable("intensity_values");
    if (intensityValueVariable == null) {
        logger.severe("Could not find variable intensity_values");
        throw (new IOException("Could not find variable intensity_values"));
    }
    assert (intensityValueVariable.getRank() == 1);
    Attribute intScaleFacAttr = intensityValueVariable.findAttribute("scale_factor");
    if (intScaleFacAttr != null) {
        intensityValueScaleFactor = intScaleFacAttr.getNumericValue().doubleValue();
    }
    // Read number of scans
    Variable scanIndexVariable = inputFile.findVariable("scan_index");
    if (scanIndexVariable == null) {
        logger.severe("Could not find variable scan_index from file " + file);
        throw (new IOException("Could not find variable scan_index from file " + file));
    }
    totalScans = scanIndexVariable.getShape()[0];
    // Read scan start positions
    // Extra element is required, because element totalScans+1 is used to
    // find the stop position for last scan
    int[] scanStartPositions = new int[totalScans + 1];
    Array scanIndexArray = null;
    try {
        scanIndexArray = scanIndexVariable.read();
    } catch (Exception e) {
        logger.severe(e.toString());
        throw (new IOException("Could not read from variable scan_index from file " + file));
    }
    IndexIterator scanIndexIterator = scanIndexArray.getIndexIterator();
    int ind = 0;
    while (scanIndexIterator.hasNext()) {
        scanStartPositions[ind] = ((Integer) scanIndexIterator.next()).intValue();
        ind++;
    }
    scanIndexIterator = null;
    scanIndexArray = null;
    scanIndexVariable = null;
    // Calc stop position for the last scan
    // This defines the end index of the last scan
    scanStartPositions[totalScans] = (int) massValueVariable.getSize();
    // Start scan RT
    double[] retentionTimes = new double[totalScans];
    Variable scanTimeVariable = inputFile.findVariable("scan_acquisition_time");
    if (scanTimeVariable == null) {
        logger.severe("Could not find variable scan_acquisition_time from file " + file);
        throw (new IOException("Could not find variable scan_acquisition_time from file " + file));
    }
    Array scanTimeArray = null;
    try {
        scanTimeArray = scanTimeVariable.read();
    } catch (Exception e) {
        logger.severe(e.toString());
        throw (new IOException("Could not read from variable scan_acquisition_time from file " + file));
    }
    IndexIterator scanTimeIterator = scanTimeArray.getIndexIterator();
    ind = 0;
    while (scanTimeIterator.hasNext()) {
        if (scanTimeVariable.getDataType().getPrimitiveClassType() == float.class) {
            retentionTimes[ind] = ((Float) scanTimeIterator.next()) / 60d;
        }
        if (scanTimeVariable.getDataType().getPrimitiveClassType() == double.class) {
            retentionTimes[ind] = ((Double) scanTimeIterator.next()) / 60d;
        }
        ind++;
    }
    // End scan RT
    // Cleanup
    scanTimeIterator = null;
    scanTimeArray = null;
    scanTimeVariable = null;
    // Fix problems caused by new QStar data converter
    // assume scan is missing when scan_index[i]<0
    // for these scans, fix variables:
    // - scan_acquisition_time: interpolate/extrapolate using times of
    // present scans
    // - scan_index: fill with following good value
    // Calculate number of good scans
    numberOfGoodScans = 0;
    for (int i = 0; i < totalScans; i++) {
        if (scanStartPositions[i] >= 0) {
            numberOfGoodScans++;
        }
    }
    // Is there need to fix something?
    if (numberOfGoodScans < totalScans) {
        // Fix scan_acquisition_time
        // - calculate average delta time between present scans
        double sumDelta = 0;
        int n = 0;
        for (int i = 0; i < totalScans; i++) {
            // Is this a present scan?
            if (scanStartPositions[i] >= 0) {
                // Yes, find next present scan
                for (int j = i + 1; j < totalScans; j++) {
                    if (scanStartPositions[j] >= 0) {
                        sumDelta += (retentionTimes[j] - retentionTimes[i]) / ((double) (j - i));
                        n++;
                        break;
                    }
                }
            }
        }
        double avgDelta = sumDelta / (double) n;
        // - fill missing scan times using nearest good scan and avgDelta
        for (int i = 0; i < totalScans; i++) {
            // Is this a missing scan?
            if (scanStartPositions[i] < 0) {
                // Yes, find nearest present scan
                int nearestI = Integer.MAX_VALUE;
                for (int j = 1; 1 < 2; j++) {
                    if ((i + j) < totalScans) {
                        if (scanStartPositions[i + j] >= 0) {
                            nearestI = i + j;
                            break;
                        }
                    }
                    if ((i - j) >= 0) {
                        if (scanStartPositions[i - j] >= 0) {
                            nearestI = i + j;
                            break;
                        }
                    }
                    // Out of bounds?
                    if (((i + j) >= totalScans) && ((i - j) < 0)) {
                        break;
                    }
                }
                if (nearestI != Integer.MAX_VALUE) {
                    retentionTimes[i] = retentionTimes[nearestI] + (i - nearestI) * avgDelta;
                } else {
                    if (i > 0) {
                        retentionTimes[i] = retentionTimes[i - 1];
                    } else {
                        retentionTimes[i] = 0;
                    }
                    logger.severe("ERROR: Could not fix incorrect QStar scan times.");
                }
            }
        }
        // Fix scanStartPositions by filling gaps with next good value
        for (int i = 0; i < totalScans; i++) {
            if (scanStartPositions[i] < 0) {
                for (int j = i + 1; j < (totalScans + 1); j++) {
                    if (scanStartPositions[j] >= 0) {
                        scanStartPositions[i] = scanStartPositions[j];
                        break;
                    }
                }
            }
        }
    }
    // Collect information about retention times, start positions and
    // lengths for scans
    scansRetentionTimes = new Hashtable<Integer, Double>();
    scansIndex = new Hashtable<Integer, Integer[]>();
    for (int i = 0; i < totalScans; i++) {
        Integer scanNum = new Integer(i);
        Integer[] startAndLength = new Integer[2];
        startAndLength[0] = scanStartPositions[i];
        startAndLength[1] = scanStartPositions[i + 1] - scanStartPositions[i];
        scansRetentionTimes.put(scanNum, new Double(retentionTimes[i]));
        scansIndex.put(scanNum, startAndLength);
    }
    scanStartPositions = null;
    retentionTimes = null;
}
Also used : Variable(ucar.nc2.Variable) Attribute(ucar.nc2.Attribute) IOException(java.io.IOException) IndexIterator(ucar.ma2.IndexIterator) IOException(java.io.IOException) DataPoint(net.sf.mzmine.datamodel.DataPoint) SimpleDataPoint(net.sf.mzmine.datamodel.impl.SimpleDataPoint) Array(ucar.ma2.Array)

Example 3 with Variable

use of org.flyte.api.v1.Variable in project mzmine2 by mzmine.

the class MassDetectionTask method run.

/**
 * @see Runnable#run()
 */
public void run() {
    // make arrays to contain everything you need
    ArrayList<Integer> pointsInScans = new ArrayList<>();
    ArrayList<Double> allMZ = new ArrayList<>();
    ArrayList<Double> allIntensities = new ArrayList<>();
    // idecies of full mass list where scan starts?
    ArrayList<Integer> startIndex = new ArrayList<>();
    ArrayList<Double> scanAcquisitionTime = new ArrayList<>();
    // XCMS needs this one
    ArrayList<Double> totalIntensity = new ArrayList<>();
    double curTotalIntensity;
    int lastPointCount = 0;
    startIndex.add(0);
    try {
        setStatus(TaskStatus.PROCESSING);
        logger.info("Started mass detector on " + dataFile);
        final Scan[] scans = scanSelection.getMatchingScans(dataFile);
        totalScans = scans.length;
        // Process scans one by one
        for (Scan scan : scans) {
            if (isCanceled())
                return;
            MassDetector detector = massDetector.getModule();
            DataPoint[] mzPeaks = detector.getMassValues(scan, massDetector.getParameterSet());
            SimpleMassList newMassList = new SimpleMassList(name, scan, mzPeaks);
            // Add new mass list to the scan
            scan.addMassList(newMassList);
            if (this.saveToCDF) {
                curTotalIntensity = 0;
                for (int a = 0; a < mzPeaks.length; a++) {
                    DataPoint curMzPeak = mzPeaks[a];
                    allMZ.add(curMzPeak.getMZ());
                    allIntensities.add(curMzPeak.getIntensity());
                    curTotalIntensity += curMzPeak.getIntensity();
                }
                scanAcquisitionTime.add(scan.getRetentionTime());
                pointsInScans.add(0);
                startIndex.add(mzPeaks.length + lastPointCount);
                totalIntensity.add(curTotalIntensity);
                lastPointCount = mzPeaks.length + lastPointCount;
            }
            processedScans++;
        }
        // Update the GUI with all new mass lists
        MZmineProjectImpl project = (MZmineProjectImpl) MZmineCore.getProjectManager().getCurrentProject();
        final RawDataTreeModel treeModel = project.getRawDataTreeModel();
        treeModel.updateGUIWithNewObjects();
        ;
        if (this.saveToCDF) {
            // ************** write mass list *******************************
            final String outFileNamePath = outFilename.getPath();
            logger.info("Saving mass detector results to netCDF file " + outFileNamePath);
            NetcdfFileWriter writer = NetcdfFileWriter.createNew(NetcdfFileWriter.Version.netcdf3, outFileNamePath, null);
            Dimension dim_massValues = writer.addDimension(null, "mass_values", allMZ.size());
            Dimension dim_intensityValues = writer.addDimension(null, "intensity_values", allIntensities.size());
            Dimension dim_scanIndex = writer.addDimension(null, "scan_index", startIndex.size() - 1);
            Dimension dim_scanAcquisitionTime = writer.addDimension(null, "scan_acquisition_time", scanAcquisitionTime.size());
            Dimension dim_totalIntensity = writer.addDimension(null, "total_intensity", totalIntensity.size());
            Dimension dim_pointsInScans = writer.addDimension(null, "point_count", pointsInScans.size());
            // add dimensions to list
            List<Dimension> dims = new ArrayList<>();
            dims.add(dim_massValues);
            dims.add(dim_intensityValues);
            dims.add(dim_scanIndex);
            dims.add(dim_scanAcquisitionTime);
            dims.add(dim_totalIntensity);
            dims.add(dim_pointsInScans);
            // make the variables that contain the actual data I think.
            Variable var_massValues = writer.addVariable(null, "mass_values", DataType.DOUBLE, "mass_values");
            Variable var_intensityValues = writer.addVariable(null, "intensity_values", DataType.DOUBLE, "intensity_values");
            Variable var_scanIndex = writer.addVariable(null, "scan_index", DataType.INT, "scan_index");
            Variable var_scanAcquisitionTime = writer.addVariable(null, "scan_acquisition_time", DataType.DOUBLE, "scan_acquisition_time");
            Variable var_totalIntensity = writer.addVariable(null, "total_intensity", DataType.DOUBLE, "total_intensity");
            Variable var_pointsInScans = writer.addVariable(null, "point_count", DataType.INT, "point_count");
            var_massValues.addAttribute(new Attribute("units", "M/Z"));
            var_intensityValues.addAttribute(new Attribute("units", "Arbitrary Intensity Units"));
            var_scanIndex.addAttribute(new Attribute("units", "index"));
            var_scanAcquisitionTime.addAttribute(new Attribute("units", "seconds"));
            var_totalIntensity.addAttribute(new Attribute("units", "Arbitrary Intensity Units"));
            var_pointsInScans.addAttribute(new Attribute("units", "count"));
            var_massValues.addAttribute(new Attribute("scale_factor", 1.0));
            var_intensityValues.addAttribute(new Attribute("scale_factor", 1.0));
            var_scanIndex.addAttribute(new Attribute("scale_factor", 1.0));
            var_scanAcquisitionTime.addAttribute(new Attribute("scale_factor", 1.0));
            var_totalIntensity.addAttribute(new Attribute("scale_factor", 1.0));
            var_pointsInScans.addAttribute(new Attribute("scale_factor", 1.0));
            // create file
            writer.create();
            ArrayDouble.D1 arr_massValues = new ArrayDouble.D1(dim_massValues.getLength());
            ArrayDouble.D1 arr_intensityValues = new ArrayDouble.D1(dim_intensityValues.getLength());
            ArrayDouble.D1 arr_scanIndex = new ArrayDouble.D1(dim_scanIndex.getLength());
            ArrayDouble.D1 arr_scanAcquisitionTime = new ArrayDouble.D1(dim_scanAcquisitionTime.getLength());
            ArrayDouble.D1 arr_totalIntensity = new ArrayDouble.D1(dim_totalIntensity.getLength());
            ArrayDouble.D1 arr_pointsInScans = new ArrayDouble.D1(dim_pointsInScans.getLength());
            for (int i = 0; i < allMZ.size(); i++) {
                arr_massValues.set(i, allMZ.get(i));
                arr_intensityValues.set(i, allIntensities.get(i));
            }
            int i = 0;
            for (; i < scanAcquisitionTime.size(); i++) {
                arr_scanAcquisitionTime.set(i, scanAcquisitionTime.get(i) * 60);
                arr_pointsInScans.set(i, pointsInScans.get(i));
                arr_scanIndex.set(i, startIndex.get(i));
                arr_totalIntensity.set(i, totalIntensity.get(i));
            }
            // arr_scanIndex.set(i,startIndex.get(i));
            // For tiny test file
            // arr_intensityValues .set(0,200);
            // arr_scanIndex .set(0,0);
            // arr_scanAcquisitionTime .set(0,10);
            // arr_totalIntensity .set(0,200);
            // arr_pointsInScans .set(0,0);
            // arr_intensityValues .set(1,300);
            // arr_scanIndex .set(1,1);
            // arr_scanAcquisitionTime .set(1,20);
            // arr_totalIntensity .set(1,300);
            // arr_pointsInScans .set(1,0);
            writer.write(var_massValues, arr_massValues);
            writer.write(var_intensityValues, arr_intensityValues);
            writer.write(var_scanIndex, arr_scanIndex);
            writer.write(var_scanAcquisitionTime, arr_scanAcquisitionTime);
            writer.write(var_totalIntensity, arr_totalIntensity);
            writer.write(var_pointsInScans, arr_pointsInScans);
            writer.close();
        }
    } catch (Exception e) {
        e.printStackTrace();
        setErrorMessage(e.getMessage());
        setStatus(TaskStatus.ERROR);
    }
    setStatus(TaskStatus.FINISHED);
    logger.info("Finished mass detector on " + dataFile);
}
Also used : Variable(ucar.nc2.Variable) Attribute(ucar.nc2.Attribute) ArrayList(java.util.ArrayList) DataPoint(net.sf.mzmine.datamodel.DataPoint) ArrayDouble(ucar.ma2.ArrayDouble) RawDataTreeModel(net.sf.mzmine.desktop.impl.projecttree.RawDataTreeModel) SimpleMassList(net.sf.mzmine.datamodel.impl.SimpleMassList) Dimension(ucar.nc2.Dimension) ArrayDouble(ucar.ma2.ArrayDouble) DataPoint(net.sf.mzmine.datamodel.DataPoint) NetcdfFileWriter(ucar.nc2.NetcdfFileWriter) Scan(net.sf.mzmine.datamodel.Scan) MZmineProjectImpl(net.sf.mzmine.project.impl.MZmineProjectImpl)

Example 4 with Variable

use of org.flyte.api.v1.Variable in project vcell by virtualcell.

the class NetCDFReader method getSavePeriod.

/**
 * Get save period, which is the time interval for saving the data. 0 dimension.
 * @return double, save period.
 */
public double getSavePeriod() throws IOException {
    Variable v = ncfile.findVariable("SaveTime");
    Double val = 0.0;
    try {
        val = v.readScalarDouble();
    } catch (IOException ioe) {
        ioe.printStackTrace(System.err);
        throw new IOException("Cannot get proper save period from the model.");
    }
    return val;
}
Also used : Variable(ucar.nc2.Variable) IOException(java.io.IOException) ArrayDouble(ucar.ma2.ArrayDouble)

Example 5 with Variable

use of org.flyte.api.v1.Variable in project vcell by virtualcell.

the class NetCDFReader method getExperimentType.

/**
 * Get experiment time.
 * ExpType = 1; This indicates that this NetCDF file is a 'Single model' file.
 * ExpType = 2; This indicates that this NetCDF file is a 'Multi-model' file.
 * @return int, experiment type.
 */
public int getExperimentType() throws IOException {
    Variable v = ncfile.findVariable("ExpType");
    int val = 0;
    try {
        val = v.readScalarInt();
    } catch (IOException ioe) {
        ioe.printStackTrace(System.err);
        throw new IOException("Cannot get proper experiment type from the model.");
    }
    return val;
}
Also used : Variable(ucar.nc2.Variable) IOException(java.io.IOException)

Aggregations

Variable (ucar.nc2.Variable)78 IOException (java.io.IOException)41 Variable (org.jpl7.Variable)33 Query (org.jpl7.Query)32 Term (org.jpl7.Term)29 Array (ucar.ma2.Array)21 Attribute (ucar.nc2.Attribute)21 Compound (org.jpl7.Compound)19 NetcdfFile (ucar.nc2.NetcdfFile)18 Map (java.util.Map)17 Dimension (ucar.nc2.Dimension)17 InvalidRangeException (ucar.ma2.InvalidRangeException)16 ArrayList (java.util.ArrayList)15 Atom (org.jpl7.Atom)15 ArrayDouble (ucar.ma2.ArrayDouble)14 File (java.io.File)10 List (java.util.List)9 ArrayFloat (ucar.ma2.ArrayFloat)9 WritableRaster (java.awt.image.WritableRaster)8 HashMap (java.util.HashMap)8