Examples with Variable org.flyte.api.v1.Variable used on opensource projects

Example 1 with Variable

use of org.flyte.api.v1.Variable in project tika by apache.

the class NetCDFParser method parse.

/*
     * (non-Javadoc)
     * 
     * @see org.apache.tika.parser.Parser#parse(java.io.InputStream,
     * org.xml.sax.ContentHandler, org.apache.tika.metadata.Metadata,
     * org.apache.tika.parser.ParseContext)
     */
public void parse(InputStream stream, ContentHandler handler, Metadata metadata, ParseContext context) throws IOException, SAXException, TikaException {
    TemporaryResources tmp = TikaInputStream.isTikaInputStream(stream) ? null : new TemporaryResources();
    TikaInputStream tis = TikaInputStream.get(stream, tmp);
    NetcdfFile ncFile = null;
    try {
        ncFile = NetcdfFile.open(tis.getFile().getAbsolutePath());
        metadata.set("File-Type-Description", ncFile.getFileTypeDescription());
        // first parse out the set of global attributes
        for (Attribute attr : ncFile.getGlobalAttributes()) {
            Property property = resolveMetadataKey(attr.getFullName());
            if (attr.getDataType().isString()) {
                metadata.add(property, attr.getStringValue());
            } else if (attr.getDataType().isNumeric()) {
                int value = attr.getNumericValue().intValue();
                metadata.add(property, String.valueOf(value));
            }
        }
        XHTMLContentHandler xhtml = new XHTMLContentHandler(handler, metadata);
        xhtml.startDocument();
        xhtml.newline();
        xhtml.element("h1", "dimensions");
        xhtml.startElement("ul");
        xhtml.newline();
        for (Dimension dim : ncFile.getDimensions()) {
            xhtml.element("li", dim.getFullName() + " = " + dim.getLength());
        }
        xhtml.endElement("ul");
        xhtml.element("h1", "variables");
        xhtml.startElement("ul");
        xhtml.newline();
        for (Variable var : ncFile.getVariables()) {
            xhtml.startElement("li");
            xhtml.characters(var.getDataType() + " " + var.getNameAndDimensions());
            xhtml.newline();
            List<Attribute> attributes = var.getAttributes();
            if (!attributes.isEmpty()) {
                xhtml.startElement("ul");
                for (Attribute element : attributes) {
                    xhtml.element("li", element.toString());
                }
                xhtml.endElement("ul");
            }
            xhtml.endElement("li");
        }
        xhtml.endElement("ul");
        xhtml.endDocument();
    } catch (IOException e) {
        throw new TikaException("NetCDF parse error", e);
    } finally {
        if (ncFile != null) {
            ncFile.close();
        }
        if (tmp != null) {
            tmp.dispose();
        }
    }
}

Example 2 with Variable

use of org.flyte.api.v1.Variable in project mzmine2 by mzmine.

the class NetCDFReadTask method startReading.

public void startReading() throws IOException {
    // Open NetCDF-file
    try {
        inputFile = NetcdfFile.open(file.getPath());
    } catch (Exception e) {
        logger.severe(e.toString());
        throw (new IOException("Couldn't open input file" + file));
    }
    /*
     * DEBUG: dump all variables for (Variable v : inputFile.getVariables()) {
     * System.out.println("variable " + v.getShortName()); }
     */
    // Find mass_values and intensity_values variables
    massValueVariable = inputFile.findVariable("mass_values");
    if (massValueVariable == null) {
        logger.severe("Could not find variable mass_values");
        throw (new IOException("Could not find variable mass_values"));
    }
    assert (massValueVariable.getRank() == 1);
    Attribute massScaleFacAttr = massValueVariable.findAttribute("scale_factor");
    if (massScaleFacAttr != null) {
        massValueScaleFactor = massScaleFacAttr.getNumericValue().doubleValue();
    }
    intensityValueVariable = inputFile.findVariable("intensity_values");
    if (intensityValueVariable == null) {
        logger.severe("Could not find variable intensity_values");
        throw (new IOException("Could not find variable intensity_values"));
    }
    assert (intensityValueVariable.getRank() == 1);
    Attribute intScaleFacAttr = intensityValueVariable.findAttribute("scale_factor");
    if (intScaleFacAttr != null) {
        intensityValueScaleFactor = intScaleFacAttr.getNumericValue().doubleValue();
    }
    // Read number of scans
    Variable scanIndexVariable = inputFile.findVariable("scan_index");
    if (scanIndexVariable == null) {
        logger.severe("Could not find variable scan_index from file " + file);
        throw (new IOException("Could not find variable scan_index from file " + file));
    }
    totalScans = scanIndexVariable.getShape()[0];
    // Read scan start positions
    // Extra element is required, because element totalScans+1 is used to
    // find the stop position for last scan
    int[] scanStartPositions = new int[totalScans + 1];
    Array scanIndexArray = null;
    try {
        scanIndexArray = scanIndexVariable.read();
    } catch (Exception e) {
        logger.severe(e.toString());
        throw (new IOException("Could not read from variable scan_index from file " + file));
    }
    IndexIterator scanIndexIterator = scanIndexArray.getIndexIterator();
    int ind = 0;
    while (scanIndexIterator.hasNext()) {
        scanStartPositions[ind] = ((Integer) scanIndexIterator.next()).intValue();
        ind++;
    }
    scanIndexIterator = null;
    scanIndexArray = null;
    scanIndexVariable = null;
    // Calc stop position for the last scan
    // This defines the end index of the last scan
    scanStartPositions[totalScans] = (int) massValueVariable.getSize();
    // Start scan RT
    double[] retentionTimes = new double[totalScans];
    Variable scanTimeVariable = inputFile.findVariable("scan_acquisition_time");
    if (scanTimeVariable == null) {
        logger.severe("Could not find variable scan_acquisition_time from file " + file);
        throw (new IOException("Could not find variable scan_acquisition_time from file " + file));
    }
    Array scanTimeArray = null;
    try {
        scanTimeArray = scanTimeVariable.read();
    } catch (Exception e) {
        logger.severe(e.toString());
        throw (new IOException("Could not read from variable scan_acquisition_time from file " + file));
    }
    IndexIterator scanTimeIterator = scanTimeArray.getIndexIterator();
    ind = 0;
    while (scanTimeIterator.hasNext()) {
        if (scanTimeVariable.getDataType().getPrimitiveClassType() == float.class) {
            retentionTimes[ind] = ((Float) scanTimeIterator.next()) / 60d;
        }
        if (scanTimeVariable.getDataType().getPrimitiveClassType() == double.class) {
            retentionTimes[ind] = ((Double) scanTimeIterator.next()) / 60d;
        }
        ind++;
    }
    // End scan RT
    // Cleanup
    scanTimeIterator = null;
    scanTimeArray = null;
    scanTimeVariable = null;
    // Fix problems caused by new QStar data converter
    // assume scan is missing when scan_index[i]<0
    // for these scans, fix variables:
    // - scan_acquisition_time: interpolate/extrapolate using times of
    // present scans
    // - scan_index: fill with following good value
    // Calculate number of good scans
    numberOfGoodScans = 0;
    for (int i = 0; i < totalScans; i++) {
        if (scanStartPositions[i] >= 0) {
            numberOfGoodScans++;
        }
    }
    // Is there need to fix something?
    if (numberOfGoodScans < totalScans) {
        // Fix scan_acquisition_time
        // - calculate average delta time between present scans
        double sumDelta = 0;
        int n = 0;
        for (int i = 0; i < totalScans; i++) {
            // Is this a present scan?
            if (scanStartPositions[i] >= 0) {
                // Yes, find next present scan
                for (int j = i + 1; j < totalScans; j++) {
                    if (scanStartPositions[j] >= 0) {
                        sumDelta += (retentionTimes[j] - retentionTimes[i]) / ((double) (j - i));
                        n++;
                        break;
                    }
                }
            }
        }
        double avgDelta = sumDelta / (double) n;
        // - fill missing scan times using nearest good scan and avgDelta
        for (int i = 0; i < totalScans; i++) {
            // Is this a missing scan?
            if (scanStartPositions[i] < 0) {
                // Yes, find nearest present scan
                int nearestI = Integer.MAX_VALUE;
                for (int j = 1; 1 < 2; j++) {
                    if ((i + j) < totalScans) {
                        if (scanStartPositions[i + j] >= 0) {
                            nearestI = i + j;
                            break;
                        }
                    }
                    if ((i - j) >= 0) {
                        if (scanStartPositions[i - j] >= 0) {
                            nearestI = i + j;
                            break;
                        }
                    }
                    // Out of bounds?
                    if (((i + j) >= totalScans) && ((i - j) < 0)) {
                        break;
                    }
                }
                if (nearestI != Integer.MAX_VALUE) {
                    retentionTimes[i] = retentionTimes[nearestI] + (i - nearestI) * avgDelta;
                } else {
                    if (i > 0) {
                        retentionTimes[i] = retentionTimes[i - 1];
                    } else {
                        retentionTimes[i] = 0;
                    }
                    logger.severe("ERROR: Could not fix incorrect QStar scan times.");
                }
            }
        }
        // Fix scanStartPositions by filling gaps with next good value
        for (int i = 0; i < totalScans; i++) {
            if (scanStartPositions[i] < 0) {
                for (int j = i + 1; j < (totalScans + 1); j++) {
                    if (scanStartPositions[j] >= 0) {
                        scanStartPositions[i] = scanStartPositions[j];
                        break;
                    }
                }
            }
        }
    }
    // Collect information about retention times, start positions and
    // lengths for scans
    scansRetentionTimes = new Hashtable<Integer, Double>();
    scansIndex = new Hashtable<Integer, Integer[]>();
    for (int i = 0; i < totalScans; i++) {
        Integer scanNum = new Integer(i);
        Integer[] startAndLength = new Integer[2];
        startAndLength[0] = scanStartPositions[i];
        startAndLength[1] = scanStartPositions[i + 1] - scanStartPositions[i];
        scansRetentionTimes.put(scanNum, new Double(retentionTimes[i]));
        scansIndex.put(scanNum, startAndLength);
    }
    scanStartPositions = null;
    retentionTimes = null;
}

Example 3 with Variable

use of org.flyte.api.v1.Variable in project mzmine2 by mzmine.

the class MassDetectionTask method run.

/**
 * @see Runnable#run()
 */
public void run() {
    // make arrays to contain everything you need
    ArrayList<Integer> pointsInScans = new ArrayList<>();
    ArrayList<Double> allMZ = new ArrayList<>();
    ArrayList<Double> allIntensities = new ArrayList<>();
    // idecies of full mass list where scan starts?
    ArrayList<Integer> startIndex = new ArrayList<>();
    ArrayList<Double> scanAcquisitionTime = new ArrayList<>();
    // XCMS needs this one
    ArrayList<Double> totalIntensity = new ArrayList<>();
    double curTotalIntensity;
    int lastPointCount = 0;
    startIndex.add(0);
    try {
        setStatus(TaskStatus.PROCESSING);
        logger.info("Started mass detector on " + dataFile);
        final Scan[] scans = scanSelection.getMatchingScans(dataFile);
        totalScans = scans.length;
        // Process scans one by one
        for (Scan scan : scans) {
            if (isCanceled())
                return;
            MassDetector detector = massDetector.getModule();
            DataPoint[] mzPeaks = detector.getMassValues(scan, massDetector.getParameterSet());
            SimpleMassList newMassList = new SimpleMassList(name, scan, mzPeaks);
            // Add new mass list to the scan
            scan.addMassList(newMassList);
            if (this.saveToCDF) {
                curTotalIntensity = 0;
                for (int a = 0; a < mzPeaks.length; a++) {
                    DataPoint curMzPeak = mzPeaks[a];
                    allMZ.add(curMzPeak.getMZ());
                    allIntensities.add(curMzPeak.getIntensity());
                    curTotalIntensity += curMzPeak.getIntensity();
                }
                scanAcquisitionTime.add(scan.getRetentionTime());
                pointsInScans.add(0);
                startIndex.add(mzPeaks.length + lastPointCount);
                totalIntensity.add(curTotalIntensity);
                lastPointCount = mzPeaks.length + lastPointCount;
            }
            processedScans++;
        }
        // Update the GUI with all new mass lists
        MZmineProjectImpl project = (MZmineProjectImpl) MZmineCore.getProjectManager().getCurrentProject();
        final RawDataTreeModel treeModel = project.getRawDataTreeModel();
        treeModel.updateGUIWithNewObjects();
        ;
        if (this.saveToCDF) {
            // ************** write mass list *******************************
            final String outFileNamePath = outFilename.getPath();
            logger.info("Saving mass detector results to netCDF file " + outFileNamePath);
            NetcdfFileWriter writer = NetcdfFileWriter.createNew(NetcdfFileWriter.Version.netcdf3, outFileNamePath, null);
            Dimension dim_massValues = writer.addDimension(null, "mass_values", allMZ.size());
            Dimension dim_intensityValues = writer.addDimension(null, "intensity_values", allIntensities.size());
            Dimension dim_scanIndex = writer.addDimension(null, "scan_index", startIndex.size() - 1);
            Dimension dim_scanAcquisitionTime = writer.addDimension(null, "scan_acquisition_time", scanAcquisitionTime.size());
            Dimension dim_totalIntensity = writer.addDimension(null, "total_intensity", totalIntensity.size());
            Dimension dim_pointsInScans = writer.addDimension(null, "point_count", pointsInScans.size());
            // add dimensions to list
            List<Dimension> dims = new ArrayList<>();
            dims.add(dim_massValues);
            dims.add(dim_intensityValues);
            dims.add(dim_scanIndex);
            dims.add(dim_scanAcquisitionTime);
            dims.add(dim_totalIntensity);
            dims.add(dim_pointsInScans);
            // make the variables that contain the actual data I think.
            Variable var_massValues = writer.addVariable(null, "mass_values", DataType.DOUBLE, "mass_values");
            Variable var_intensityValues = writer.addVariable(null, "intensity_values", DataType.DOUBLE, "intensity_values");
            Variable var_scanIndex = writer.addVariable(null, "scan_index", DataType.INT, "scan_index");
            Variable var_scanAcquisitionTime = writer.addVariable(null, "scan_acquisition_time", DataType.DOUBLE, "scan_acquisition_time");
            Variable var_totalIntensity = writer.addVariable(null, "total_intensity", DataType.DOUBLE, "total_intensity");
            Variable var_pointsInScans = writer.addVariable(null, "point_count", DataType.INT, "point_count");
            var_massValues.addAttribute(new Attribute("units", "M/Z"));
            var_intensityValues.addAttribute(new Attribute("units", "Arbitrary Intensity Units"));
            var_scanIndex.addAttribute(new Attribute("units", "index"));
            var_scanAcquisitionTime.addAttribute(new Attribute("units", "seconds"));
            var_totalIntensity.addAttribute(new Attribute("units", "Arbitrary Intensity Units"));
            var_pointsInScans.addAttribute(new Attribute("units", "count"));
            var_massValues.addAttribute(new Attribute("scale_factor", 1.0));
            var_intensityValues.addAttribute(new Attribute("scale_factor", 1.0));
            var_scanIndex.addAttribute(new Attribute("scale_factor", 1.0));
            var_scanAcquisitionTime.addAttribute(new Attribute("scale_factor", 1.0));
            var_totalIntensity.addAttribute(new Attribute("scale_factor", 1.0));
            var_pointsInScans.addAttribute(new Attribute("scale_factor", 1.0));
            // create file
            writer.create();
            ArrayDouble.D1 arr_massValues = new ArrayDouble.D1(dim_massValues.getLength());
            ArrayDouble.D1 arr_intensityValues = new ArrayDouble.D1(dim_intensityValues.getLength());
            ArrayDouble.D1 arr_scanIndex = new ArrayDouble.D1(dim_scanIndex.getLength());
            ArrayDouble.D1 arr_scanAcquisitionTime = new ArrayDouble.D1(dim_scanAcquisitionTime.getLength());
            ArrayDouble.D1 arr_totalIntensity = new ArrayDouble.D1(dim_totalIntensity.getLength());
            ArrayDouble.D1 arr_pointsInScans = new ArrayDouble.D1(dim_pointsInScans.getLength());
            for (int i = 0; i < allMZ.size(); i++) {
                arr_massValues.set(i, allMZ.get(i));
                arr_intensityValues.set(i, allIntensities.get(i));
            }
            int i = 0;
            for (; i < scanAcquisitionTime.size(); i++) {
                arr_scanAcquisitionTime.set(i, scanAcquisitionTime.get(i) * 60);
                arr_pointsInScans.set(i, pointsInScans.get(i));
                arr_scanIndex.set(i, startIndex.get(i));
                arr_totalIntensity.set(i, totalIntensity.get(i));
            }
            // arr_scanIndex.set(i,startIndex.get(i));
            // For tiny test file
            // arr_intensityValues .set(0,200);
            // arr_scanIndex .set(0,0);
            // arr_scanAcquisitionTime .set(0,10);
            // arr_totalIntensity .set(0,200);
            // arr_pointsInScans .set(0,0);
            // arr_intensityValues .set(1,300);
            // arr_scanIndex .set(1,1);
            // arr_scanAcquisitionTime .set(1,20);
            // arr_totalIntensity .set(1,300);
            // arr_pointsInScans .set(1,0);
            writer.write(var_massValues, arr_massValues);
            writer.write(var_intensityValues, arr_intensityValues);
            writer.write(var_scanIndex, arr_scanIndex);
            writer.write(var_scanAcquisitionTime, arr_scanAcquisitionTime);
            writer.write(var_totalIntensity, arr_totalIntensity);
            writer.write(var_pointsInScans, arr_pointsInScans);
            writer.close();
        }
    } catch (Exception e) {
        e.printStackTrace();
        setErrorMessage(e.getMessage());
        setStatus(TaskStatus.ERROR);
    }
    setStatus(TaskStatus.FINISHED);
    logger.info("Finished mass detector on " + dataFile);
}

Example 4 with Variable

use of org.flyte.api.v1.Variable in project vcell by virtualcell.

the class NetCDFReader method getSavePeriod.

/**
 * Get save period, which is the time interval for saving the data. 0 dimension.
 * @return double, save period.
 */
public double getSavePeriod() throws IOException {
    Variable v = ncfile.findVariable("SaveTime");
    Double val = 0.0;
    try {
        val = v.readScalarDouble();
    } catch (IOException ioe) {
        ioe.printStackTrace(System.err);
        throw new IOException("Cannot get proper save period from the model.");
    }
    return val;
}

Example 5 with Variable

use of org.flyte.api.v1.Variable in project vcell by virtualcell.

the class NetCDFReader method getExperimentType.

/**
 * Get experiment time.
 * ExpType = 1; This indicates that this NetCDF file is a 'Single model' file.
 * ExpType = 2; This indicates that this NetCDF file is a 'Multi-model' file.
 * @return int, experiment type.
 */
public int getExperimentType() throws IOException {
    Variable v = ncfile.findVariable("ExpType");
    int val = 0;
    try {
        val = v.readScalarInt();
    } catch (IOException ioe) {
        ioe.printStackTrace(System.err);
        throw new IOException("Cannot get proper experiment type from the model.");
    }
    return val;
}