Examples with Index com.google.datastore.admin.v1.Index used on opensource projects

Example 11 with Index

use of com.google.datastore.admin.v1.Index in project mzmine2 by mzmine.

the class NetCDFReadTask method readNextScan.

/**
 * Reads one scan from the file. Requires that general information has already been read.
 */
private Scan readNextScan() throws IOException {
    // Get scan starting position and length
    int[] scanStartPosition = new int[1];
    int[] scanLength = new int[1];
    Integer[] startAndLength = scansIndex.get(scanNum);
    // End of file
    if (startAndLength == null) {
        return null;
    }
    scanStartPosition[0] = startAndLength[0];
    scanLength[0] = startAndLength[1];
    // Get retention time of the scan
    Double retentionTime = scansRetentionTimes.get(scanNum);
    if (retentionTime == null) {
        logger.severe("Could not find retention time for scan " + scanNum);
        throw (new IOException("Could not find retention time for scan " + scanNum));
    }
    // An empty scan needs special attention..
    if (scanLength[0] == 0) {
        scanNum++;
        return new SimpleScan(null, scanNum, 1, retentionTime.doubleValue(), 0, 0, null, new DataPoint[0], MassSpectrumType.CENTROIDED, PolarityType.UNKNOWN, "", null);
    }
    // Is there any way how to extract polarity from netcdf?
    PolarityType polarity = PolarityType.UNKNOWN;
    // Is there any way how to extract scan definition from netcdf?
    String scanDefinition = "";
    // Read mass and intensity values
    Array massValueArray;
    Array intensityValueArray;
    try {
        massValueArray = massValueVariable.read(scanStartPosition, scanLength);
        intensityValueArray = intensityValueVariable.read(scanStartPosition, scanLength);
    } catch (Exception e) {
        logger.log(Level.SEVERE, "Could not read from variables mass_values and/or intensity_values.", e);
        throw (new IOException("Could not read from variables mass_values and/or intensity_values."));
    }
    Index massValuesIndex = massValueArray.getIndex();
    Index intensityValuesIndex = intensityValueArray.getIndex();
    int arrayLength = massValueArray.getShape()[0];
    DataPoint[] dataPoints = new DataPoint[arrayLength];
    for (int j = 0; j < arrayLength; j++) {
        Index massIndex0 = massValuesIndex.set0(j);
        Index intensityIndex0 = intensityValuesIndex.set0(j);
        double mz = massValueArray.getDouble(massIndex0) * massValueScaleFactor;
        double intensity = intensityValueArray.getDouble(intensityIndex0) * intensityValueScaleFactor;
        dataPoints[j] = new SimpleDataPoint(mz, intensity);
    }
    scanNum++;
    // Auto-detect whether this scan is centroided
    MassSpectrumType spectrumType = ScanUtils.detectSpectrumType(dataPoints);
    SimpleScan buildingScan = new SimpleScan(null, scanNum, 1, retentionTime.doubleValue(), 0, 0, null, dataPoints, spectrumType, polarity, scanDefinition, null);
    return buildingScan;
}

Example 12 with Index

use of com.google.datastore.admin.v1.Index in project imageio-ext by geosolutions-it.

the class HOPSConverter method setTimeVariableAttributes.

// ////////////////////////////////////////////////////////////////////////
// 
// HELPERS !!!
// 
// ////////////////////////////////////////////////////////////////////////
private float setTimeVariableAttributes(Variable time_0, NetcdfFileWriteable ncFileOut) throws IOException {
    Array time_0_Data = time_0.read();
    Index time_0_Index = time_0_Data.getIndex();
    final String name = time_0.getName();
    final float referenceTime = time_0_Data.getFloat(time_0_Index.set(0));
    float fTime = referenceTime;
    Attribute offset = time_0.findAttribute("add_offset");
    if (offset != null)
        fTime += offset.getNumericValue().floatValue();
    GregorianCalendar calendar = null;
    if (time_0.getDescription().toLowerCase().contains("modified julian")) {
        calendar = NetCDFConverterUtilities.fromModifiedJulian(fTime, time_0.getDescription(), time_0.getUnitsString());
    } else {
        calendar = NetCDFConverterUtilities.fromJulian(fTime);
    }
    final String year = Integer.toString(calendar.get(Calendar.YEAR));
    final String month = Integer.toString((calendar.get(Calendar.MONTH) + 1));
    final String day = Integer.toString(calendar.get(Calendar.DAY_OF_MONTH));
    String hour = Integer.toString(calendar.get(Calendar.HOUR));
    if (hour.equalsIgnoreCase("0"))
        hour += "0";
    String minute = Integer.toString(calendar.get(Calendar.MINUTE));
    if (minute.equalsIgnoreCase("0"))
        minute += "0";
    String second = Integer.toString(calendar.get(Calendar.SECOND));
    if (second.equalsIgnoreCase("0"))
        second += "0";
    final String millisecond = Integer.toString(calendar.get(Calendar.MILLISECOND));
    final StringBuffer sbTime = new StringBuffer(year).append("-").append(month).append("-").append(day).append(" ").append(hour).append(":").append(minute).append(":").append(second).append(".").append(millisecond);
    ncFileOut.addVariableAttribute(name, "units", "days since " + sbTime.toString());
    ncFileOut.addVariableAttribute(name, "long_name", "time");
    return referenceTime;
}

Example 13 with Index

use of com.google.datastore.admin.v1.Index in project imageio-ext by geosolutions-it.

the class HOPSConverter method runTest.

public void runTest() throws IOException {
    try {
        final NetcdfFile ncFileIn = NetcdfFile.open(inputFile.getAbsolutePath());
        final File outDir = /*createTodayDirectory(this.outDir)*/
        new File("C:/work/data/testWritten/");
        // keep original name
        final File outputFile = File.createTempFile(inputFile.getName(), ".tmp");
        final NetcdfFileWriteable ncFileOut = NetcdfFileWriteable.createNew(outputFile.getAbsolutePath());
        // input dimensions
        Dimension timeDim0 = ncFileIn.findDimension("time");
        Dimension latDim0 = ncFileIn.findDimension("lat");
        Dimension lonDim0 = ncFileIn.findDimension("lon");
        // input variables
        Variable time_0 = ncFileIn.findVariable("time");
        Array time_0_Data = time_0.read();
        Index time_0_Index = time_0_Data.getIndex();
        Variable lon_0 = ncFileIn.findVariable("lon");
        final int nLon = lon_0.getDimension(0).getLength();
        Variable lat_0 = ncFileIn.findVariable("lat");
        final int nLat = lat_0.getDimension(0).getLength();
        Array lat_0_Data = lat_0.read();
        Index lat_0_Index = lat_0_Data.getIndex();
        Array lon_0_Data = lon_0.read();
        Index lon_0_Index = lon_0_Data.getIndex();
        // Depth
        Variable z_0 = ncFileIn.findVariable("z");
        final int nLevels = z_0.getDimension(0).getLength();
        Array z_0_Data = z_0.read();
        Index z_0_Index = z_0_Data.getIndex();
        Dimension timeDim = ncFileOut.addDimension("time", timeDim0.getLength());
        Dimension latDim = ncFileOut.addDimension(NetCDFUtilities.LAT, latDim0.getLength());
        Dimension lonDim = ncFileOut.addDimension(NetCDFUtilities.LON, lonDim0.getLength());
        Dimension depthDim = ncFileOut.addDimension(NetCDFUtilities.DEPTH, nLevels);
        // writing file
        computeMatrixExtremes(lat_0_Data, lon_0_Data, lonDim0.getLength(), latDim0.getLength(), lat_0_Index, lon_0_Index);
        NetCDFConverterUtilities.copyGlobalAttributes(ncFileOut, ncFileIn.getGlobalAttributes());
        // //
        // 
        // Time requires a special Management
        // 
        // //
        // time Variable
        ncFileOut.addVariable("time", DataType.FLOAT, new Dimension[] { timeDim });
        // ncFileOut.addVariableAttribute("Time", "long_name", "Time");
        final float referenceTime = setTimeVariableAttributes(time_0, ncFileOut);
        // lat Variable
        ArrayFloat lat_1_Data = new ArrayFloat(new int[] { latDim0.getLength() });
        Index lat_1_Index = lat_1_Data.getIndex();
        ncFileOut.addVariable(NetCDFUtilities.LAT, DataType.FLOAT, new Dimension[] { latDim });
        ncFileOut.addVariableAttribute(NetCDFUtilities.LAT, "long_name", NetCDFUtilities.LATITUDE);
        ncFileOut.addVariableAttribute(NetCDFUtilities.LAT, UNITS, lat_0.getUnitsString());
        for (int yPos = 0; yPos < latDim0.getLength(); yPos++) {
            lat_1_Data.setFloat(lat_1_Index.set(yPos), new Float(this.ymax - (new Float(yPos).floatValue() * this.periodY)).floatValue());
        // new Float(
        // this.ymin
        // + (new Float(yPos)
        // .floatValue() * this.periodY))
        // .floatValue());
        }
        // lon Variable
        ArrayFloat lon_1_Data = new ArrayFloat(new int[] { lonDim0.getLength() });
        Index lon_1_Index = lon_1_Data.getIndex();
        ncFileOut.addVariable(NetCDFUtilities.LON, DataType.FLOAT, new Dimension[] { lonDim });
        ncFileOut.addVariableAttribute(NetCDFUtilities.LON, "long_name", NetCDFUtilities.LONGITUDE);
        ncFileOut.addVariableAttribute(NetCDFUtilities.LON, UNITS, lon_0.getUnitsString());
        for (int xPos = 0; xPos < lonDim0.getLength(); xPos++) {
            lon_1_Data.setFloat(lon_1_Index.set(xPos), new Float(this.xmin + (new Float(xPos).floatValue() * this.periodX)).floatValue());
        }
        // depth level Variable
        ArrayFloat depthlevelDim_1_Data = new ArrayFloat(new int[] { depthDim.getLength() });
        Index depthlevelDim_1_Index = depthlevelDim_1_Data.getIndex();
        ncFileOut.addVariable(NetCDFUtilities.DEPTH, DataType.FLOAT, new Dimension[] { depthDim });
        ncFileOut.addVariableAttribute(NetCDFUtilities.DEPTH, "long_name", NetCDFUtilities.DEPTH);
        ncFileOut.addVariableAttribute(NetCDFUtilities.DEPTH, UNITS, z_0.getUnitsString());
        ncFileOut.addVariableAttribute(NetCDFUtilities.DEPTH, "positive", "up");
        for (int wPos = 0; wPos < depthDim.getLength(); wPos++) {
            depthlevelDim_1_Data.setFloat(depthlevelDim_1_Index.set(wPos), z_0_Data.getFloat(depthlevelDim_1_Index));
        }
        // {} Variables
        for (int i = 0; i < NUMVARS; i++) {
            String varName = (String) variables.get(i);
            Variable var = ncFileIn.findVariable(varName);
            ncFileOut.addVariable(varName, var.getDataType(), new Dimension[] { timeDim, depthDim, latDim, lonDim });
            NetCDFConverterUtilities.setVariableAttributes(var, ncFileOut, new String[] { "positions" });
        }
        // writing bin data ...
        ncFileOut.create();
        ArrayFloat timeData = new ArrayFloat(new int[] { timeDim.getLength() });
        Index timeIndex = timeData.getIndex();
        for (int t = 0; t < timeDim.getLength(); t++) {
            float julianTime = time_0_Data.getFloat(time_0_Index.set(t)) - referenceTime;
            timeData.setFloat(timeIndex.set(t), julianTime);
        }
        ncFileOut.write("time", timeData);
        Variable timeVar = ncFileOut.findVariable("time");
        timeDim.addCoordinateVariable(timeVar);
        ncFileOut.write(NetCDFUtilities.LAT, lat_1_Data);
        ncFileOut.write(NetCDFUtilities.LON, lon_1_Data);
        Variable depthVar = ncFileOut.findVariable("depth");
        depthDim.addCoordinateVariable(depthVar);
        ncFileOut.write(NetCDFUtilities.DEPTH, depthlevelDim_1_Data);
        // TODO: AutoApply MASK?
        ArrayFloat maskMatrix = new ArrayFloat.D2(latDim.getLength(), lonDim.getLength());
        Index maskIma = maskMatrix.getIndex();
        if (APPLY_MASK) {
            Variable mask = ncFileIn.findVariable("mask");
            Array maskData = mask.read();
            Index maskIndex = maskData.getIndex();
            ArrayFloat tempData = new ArrayFloat(new int[] { latDim0.getLength(), lonDim0.getLength() });
            Index tempIndex = tempData.getIndex();
            for (int yPos = 0; yPos < latDim0.getLength(); yPos++) {
                for (int xPos = 0; xPos < lonDim0.getLength(); xPos++) {
                    tempData.setFloat(tempIndex.set(yPos, xPos), maskData.getFloat(maskIndex.set(yPos, xPos)));
                }
            }
            WritableRaster outData = Resampler(lat_0_Data, lon_0_Data, lonDim0.getLength(), latDim0.getLength(), 2, tempData, -1);
            for (int j = 0; j < latDim0.getLength(); j++) {
                for (int k = 0; k < lonDim0.getLength(); k++) {
                    float sample = outData.getSampleFloat(k, j, 0);
                    maskMatrix.setFloat(maskIma.set(j, k), sample);
                }
            }
        }
        for (int i = 0; i < NUMVARS; i++) {
            String varName = (String) variables.get(i);
            Variable var = ncFileIn.findVariable(varName);
            Array originalVarData = var.read();
            Index varIndex = originalVarData.getIndex();
            Attribute fv = var.findAttribute("_FillValue");
            float fillValue = Float.NaN;
            if (fv != null) {
                fillValue = (fv.getNumericValue()).floatValue();
            }
            ArrayFloat T_tmp_Data = new ArrayFloat(new int[] { latDim0.getLength(), lonDim0.getLength() });
            Index T_tmp_Index = T_tmp_Data.getIndex();
            ArrayFloat Tmatrix = new ArrayFloat.D4(timeDim.getLength(), depthDim.getLength(), latDim.getLength(), lonDim.getLength());
            Index Tima = Tmatrix.getIndex();
            for (int tPos = 0; tPos < timeDim0.getLength(); tPos++) {
                for (int levelPos = 0; levelPos < depthDim.getLength(); levelPos++) {
                    for (int yPos = 0; yPos < latDim0.getLength(); yPos++) {
                        for (int xPos = 0; xPos < lonDim0.getLength(); xPos++) {
                            T_tmp_Data.setFloat(T_tmp_Index.set(yPos, xPos), originalVarData.getFloat(varIndex.set(tPos, yPos, xPos, levelPos)));
                        }
                    }
                    WritableRaster outData = Resampler(lat_0_Data, lon_0_Data, lonDim0.getLength(), latDim0.getLength(), 2, T_tmp_Data, fillValue);
                    for (int j = 0; j < latDim0.getLength(); j++) {
                        for (int k = 0; k < lonDim0.getLength(); k++) {
                            float sample = outData.getSampleFloat(k, j, 0);
                            if (APPLY_MASK) {
                                float maskValue = maskMatrix.getFloat(maskIma.set(j, k));
                                if (maskValue == 0)
                                    sample = fillValue;
                            }
                            Tmatrix.setFloat(Tima.set(tPos, levelPos, j, k), sample);
                        }
                    }
                }
            }
            ncFileOut.write(varName, Tmatrix);
        }
        ncFileOut.close();
        outputFile.renameTo(new File(outDir + File.separator + inputFile.getName()));
    } catch (Exception e) {
        // something bad happened
        if (LOGGER.isLoggable(Level.INFO))
            LOGGER.log(Level.INFO, e.getLocalizedMessage(), e);
        JAI.getDefaultInstance().getTileCache().flush();
    }
}

Example 14 with Index

use of com.google.datastore.admin.v1.Index in project imageio-ext by geosolutions-it.

the class InterpolateVNetCDF method run.

private void run() {
    try {
        NetcdfFile ncFileIn = NetcdfFile.open(fileNameIn);
        ncFileIn.writeCDL(System.out, true);
        NetcdfFileWriteable ncFileOut = NetcdfFileWriteable.createNew(fileNameOut);
        NetcdfFileWriteable ncFileOutV = NetcdfFileWriteable.createNew(fileNameOutV);
        File iniFile = new File("D:/tmp/netcdf/pi_ini.in");
        // //
        // 
        // Determining Initialization Time
        // 
        // //
        float julianDay = Float.NaN;
        if (iniFile.exists()) {
            FileImageInputStream fiis = new FileImageInputStream(iniFile);
            boolean goOn = true;
            while (goOn) {
                try {
                    String line = fiis.readLine();
                    if (line.contains("TSTART")) {
                        final int startIndex = line.indexOf("TSTART");
                        String time = line.substring(0, startIndex).trim();
                        julianDay = Float.parseFloat(time);
                        julianDay += 2440000;
                        break;
                    }
                } catch (IOException ioe) {
                    goOn = false;
                }
            }
        }
        GregorianCalendar calendar;
        if (!Float.isNaN(julianDay)) {
            calendar = NetCDFConverterUtilities.fromJulian(julianDay);
        } else
            calendar = new GregorianCalendar();
        // input dimensions
        Dimension timeDim0 = ncFileIn.findDimension("time");
        Dimension tLatDim0 = ncFileIn.findDimension("tlat");
        Dimension tLonDim0 = ncFileIn.findDimension("tlon");
        Dimension vLatDim0 = ncFileIn.findDimension("vlat");
        Dimension vLonDim0 = ncFileIn.findDimension("vlon");
        // input variablesT
        Variable time_0 = ncFileIn.findVariable("time");
        Array time_0_Data = time_0.read();
        Index time_0_Index = time_0_Data.getIndex();
        Variable tGrid3 = ncFileIn.findVariable("tgrid3");
        Variable vGrid3 = ncFileIn.findVariable("vgrid3");
        // //
        // 
        // Handling tracers-related objects
        // 
        // //
        final int nTLat = tGrid3.getDimension(0).getLength();
        final int nTLon = tGrid3.getDimension(1).getLength();
        final int nTLevels = tGrid3.getDimension(2).getLength();
        final String tUnits = ((Attribute) tGrid3.findAttribute(UNITS)).getStringValue();
        final String[] tUnit = tUnits.split(",");
        // //
        // 
        // Getting tracers grid lat/lon/depth
        // 
        // //
        Array tLatOriginalData = tGrid3.read("0:" + (nTLat - 1) + ":1, 0:0:1, 0:0:1, 1:1:1").reduce();
        Index tLatOriginalIndex = tLatOriginalData.getIndex();
        Array tLonOriginalData = tGrid3.read("0:0:1, 0:" + (nTLon - 1) + ":1, 0:0:1, 0:0:1").reduce();
        Index tLonOriginalIndex = tLonOriginalData.getIndex();
        Array tLevOriginalData = tGrid3.read("0:" + (nTLat - 1) + ":1, 0:" + (nTLon - 1) + ":1, 0:" + (nTLevels - 1) + ":1, 2:2:1").reduce();
        Index tLevOriginalIndex = tLevOriginalData.getIndex();
        // //
        // 
        // Handling velocity-related objects
        // 
        // //
        final int nVLat = vGrid3.getDimension(0).getLength();
        final int nVLon = vGrid3.getDimension(1).getLength();
        final int nVLevels = vGrid3.getDimension(2).getLength();
        final String vUnits = ((Attribute) vGrid3.findAttribute(UNITS)).getStringValue();
        final String[] vUnit = tUnits.split(",");
        // //
        // 
        // Getting velocity grid lat/lon/depth
        // 
        // //
        Array vLatOriginalData = vGrid3.read("0:" + (nVLat - 1) + ":1, 0:0:1, 0:0:1, 1:1:1").reduce();
        Index vLatOriginalIndex = vLatOriginalData.getIndex();
        Array vLonOriginalData = vGrid3.read("0:0:1, 0:" + (nVLon - 1) + ":1, 0:0:1, 0:0:1").reduce();
        Index vLonOriginalIndex = vLonOriginalData.getIndex();
        Array vLevOriginalData = vGrid3.read("0:" + (nVLat - 1) + ":1, 0:" + (nVLon - 1) + ":1, 0:" + (nVLevels - 1) + ":1, 2:2:1").reduce();
        Index vLevOriginalIndex = vLevOriginalData.getIndex();
        // //
        // 
        // Adding dimensions to the first dataset
        // 
        // //
        Dimension timeDimT = ncFileOut.addDimension("time", timeDim0.getLength());
        Dimension latDimT = ncFileOut.addDimension(NetCDFUtilities.LAT, tLatDim0.getLength());
        Dimension lonDimT = ncFileOut.addDimension(NetCDFUtilities.LON, tLonDim0.getLength());
        Dimension depthDimT = ncFileOut.addDimension(NetCDFUtilities.DEPTH, nTLevels);
        // //
        // 
        // Adding dimensions to the second dataset
        // 
        // //
        Dimension timeDimV = ncFileOutV.addDimension("time", timeDim0.getLength());
        Dimension latDimV = ncFileOutV.addDimension(NetCDFUtilities.LAT, vLatDim0.getLength());
        Dimension lonDimV = ncFileOutV.addDimension(NetCDFUtilities.LON, vLonDim0.getLength());
        Dimension depthDimV = ncFileOutV.addDimension(NetCDFUtilities.DEPTH, nVLevels);
        // //
        // 
        // Computing TGrid extremes
        // 
        // //
        computeMatrixExtremes(tLatOriginalData, tLonOriginalData, tLevOriginalData, tLonDim0.getLength(), tLatDim0.getLength(), depthDimT.getLength(), tLatOriginalIndex, tLonOriginalIndex, tLevOriginalIndex, T);
        // //
        // 
        // Computing VGrid extremes
        // 
        // //
        computeMatrixExtremes(vLatOriginalData, vLonOriginalData, vLevOriginalData, vLonDim0.getLength(), vLatDim0.getLength(), depthDimV.getLength(), vLatOriginalIndex, vLonOriginalIndex, vLevOriginalIndex, V);
        // //
        // 
        // Setting global attributes
        // 
        // //
        List globalAttributes = ncFileIn.getGlobalAttributes();
        copyGlobalAttributes(ncFileOut, globalAttributes);
        copyGlobalAttributes(ncFileOutV, globalAttributes);
        // //
        // 
        // Adding coord vars to the output files
        // 
        // //
        // time variable
        ncFileOut.addVariable("time", DataType.FLOAT, new Dimension[] { timeDimT });
        ncFileOutV.addVariable("time", DataType.FLOAT, new Dimension[] { timeDimV });
        setTimeVariableAttributes(time_0, ncFileOut, calendar);
        setTimeVariableAttributes(time_0, ncFileOutV, calendar);
        // ////////////////////////////////////////////////////////////////
        // 
        // Adding coord vars to the first dataset
        // 
        // ////////////////////////////////////////////////////////////////
        // lat Variable
        ArrayFloat tLatData = new ArrayFloat(new int[] { tLatDim0.getLength() });
        Index tLatIndex = tLatData.getIndex();
        ncFileOut.addVariable(NetCDFUtilities.LAT, DataType.FLOAT, new Dimension[] { latDimT });
        ncFileOut.addVariableAttribute(NetCDFUtilities.LAT, "long_name", LATITUDE);
        ncFileOut.addVariableAttribute(NetCDFUtilities.LAT, UNITS, tUnit[1].trim());
        for (int yPos = 0; yPos < tLatDim0.getLength(); yPos++) {
            tLatData.setFloat(tLatIndex.set(yPos), new Float(this.ymaxTV[T] - (new Float(yPos).floatValue() * this.yperiodTV[T])).floatValue());
        }
        // lon Variable
        ArrayFloat tLonData = new ArrayFloat(new int[] { tLonDim0.getLength() });
        Index tLonIndex = tLonData.getIndex();
        ncFileOut.addVariable(NetCDFUtilities.LON, DataType.FLOAT, new Dimension[] { lonDimT });
        ncFileOut.addVariableAttribute(NetCDFUtilities.LON, "long_name", LONGITUDE);
        ncFileOut.addVariableAttribute(NetCDFUtilities.LON, UNITS, tUnit[0].trim());
        for (int xPos = 0; xPos < tLonDim0.getLength(); xPos++) {
            tLonData.setFloat(tLonIndex.set(xPos), new Float(this.xminTV[T] + (new Float(xPos).floatValue() * this.xperiodTV[T])).floatValue());
        }
        // depth level Variable
        ArrayFloat tDepthData = new ArrayFloat(new int[] { depthDimT.getLength() });
        Index tDepthIndex = tDepthData.getIndex();
        ncFileOut.addVariable(NetCDFUtilities.DEPTH, DataType.FLOAT, new Dimension[] { depthDimT });
        ncFileOut.addVariableAttribute(DEPTH, "long_name", DEPTH);
        ncFileOut.addVariableAttribute(DEPTH, UNITS, tUnit[2].trim());
        for (int zPos = 0; zPos < depthDimT.getLength(); zPos++) {
            tDepthData.setFloat(tDepthIndex.set(zPos), new Float(this.zmaxTV[T] - (new Float(zPos).floatValue() * this.zperiodTV[T])).floatValue());
        }
        // ////////////////////////////////////////////////////////////////
        // 
        // Adding coord vars to the second dataset
        // 
        // ////////////////////////////////////////////////////////////////
        // lat Variable
        ArrayFloat vLatData = new ArrayFloat(new int[] { tLatDim0.getLength() });
        Index vLatIndex = vLatData.getIndex();
        ncFileOutV.addVariable(NetCDFUtilities.LAT, DataType.FLOAT, new Dimension[] { latDimV });
        ncFileOutV.addVariableAttribute(NetCDFUtilities.LAT, "long_name", LATITUDE);
        ncFileOutV.addVariableAttribute(NetCDFUtilities.LAT, UNITS, vUnit[1].trim());
        for (int yPos = 0; yPos < vLatDim0.getLength(); yPos++) {
            vLatData.setFloat(vLatIndex.set(yPos), new Float(this.ymaxTV[V] - (new Float(yPos).floatValue() * this.yperiodTV[V])).floatValue());
        }
        // lon Variable
        ArrayFloat vLonData = new ArrayFloat(new int[] { tLonDim0.getLength() });
        Index vLonIndex = vLonData.getIndex();
        ncFileOutV.addVariable(NetCDFUtilities.LON, DataType.FLOAT, new Dimension[] { lonDimV });
        ncFileOutV.addVariableAttribute(NetCDFUtilities.LON, "long_name", LONGITUDE);
        ncFileOutV.addVariableAttribute(NetCDFUtilities.LON, UNITS, vUnit[0].trim());
        for (int xPos = 0; xPos < vLonDim0.getLength(); xPos++) {
            vLonData.setFloat(vLonIndex.set(xPos), new Float(this.xminTV[V] + (new Float(xPos).floatValue() * this.xperiodTV[V])).floatValue());
        }
        // depth level Variable
        ArrayFloat vDepthData = new ArrayFloat(new int[] { depthDimT.getLength() });
        Index vDepthIndex = vDepthData.getIndex();
        ncFileOutV.addVariable(NetCDFUtilities.DEPTH, DataType.FLOAT, new Dimension[] { depthDimV });
        ncFileOutV.addVariableAttribute(DEPTH, "long_name", DEPTH);
        ncFileOutV.addVariableAttribute(DEPTH, UNITS, vUnit[2].trim());
        for (int zPos = 0; zPos < depthDimV.getLength(); zPos++) {
            vDepthData.setFloat(vDepthIndex.set(zPos), new Float(this.zmaxTV[V] - (new Float(zPos).floatValue() * this.zperiodTV[V])).floatValue());
        }
        // Tracers related Variables
        for (int i = 0; i < NUMVARS_T; i++) {
            String varName = variablesT.get(i);
            Variable var = ncFileIn.findVariable(varName);
            ncFileOut.addVariable(varName, var.getDataType(), new Dimension[] { timeDimT, depthDimT, latDimT, lonDimT });
            setVariableAttributes(var, ncFileOut, new String[] { "positions" });
        }
        // Velocity related Variables
        for (int i = 0; i < NUMVARS_V; i++) {
            String varName = variablesV.get(i);
            Variable var = ncFileIn.findVariable(varName);
            ncFileOutV.addVariable(varName + "0", var.getDataType(), new Dimension[] { timeDimV, depthDimV, latDimV, lonDimV });
            ncFileOutV.addVariable(varName + "1", var.getDataType(), new Dimension[] { timeDimV, depthDimV, latDimV, lonDimV });
            setVariableAttributes(var, ncFileOutV, new String[] { "positions" }, 0);
            setVariableAttributes(var, ncFileOutV, new String[] { "positions" }, 1);
        }
        // writing bin data ...
        ncFileOut.create();
        ncFileOutV.create();
        ArrayFloat timeData = new ArrayFloat(new int[] { timeDimT.getLength() });
        Index timeIndex = timeData.getIndex();
        for (int t = 0; t < timeDimT.getLength(); t++) {
            float seconds = time_0_Data.getFloat(time_0_Index.set(t));
            timeData.setFloat(timeIndex.set(t), seconds);
        }
        // //
        // 
        // Writing coordVars for the first dataset
        // 
        // //
        ncFileOut.write("time", timeData);
        Variable timeVar = ncFileOut.findVariable("time");
        timeDimT.addCoordinateVariable(timeVar);
        Variable depthVarT = ncFileOut.findVariable("depth");
        depthDimT.addCoordinateVariable(depthVarT);
        ncFileOut.write(NetCDFUtilities.DEPTH, vDepthData);
        ncFileOut.write(NetCDFUtilities.LAT, tLatData);
        ncFileOut.write(NetCDFUtilities.LON, tLonData);
        // //
        // 
        // Writing coordVars for the second dataset
        // 
        // //
        ncFileOutV.write("time", timeData);
        Variable timeVarV = ncFileOutV.findVariable("time");
        timeDimV.addCoordinateVariable(timeVarV);
        Variable depthVarV = ncFileOut.findVariable("depth");
        depthDimV.addCoordinateVariable(depthVarV);
        ncFileOutV.write(NetCDFUtilities.DEPTH, vDepthData);
        ncFileOutV.write(NetCDFUtilities.LAT, vLatData);
        ncFileOutV.write(NetCDFUtilities.LON, vLonData);
        // TODO: AutoApply MASK?
        ArrayFloat maskMatrixT = new ArrayFloat.D2(latDimT.getLength(), lonDimT.getLength());
        Index maskImaT = maskMatrixT.getIndex();
        if (APPLY_MASK) {
            Variable mask = ncFileIn.findVariable("landt");
            Array maskData = mask.read();
            Index maskIndex = maskData.getIndex();
            ArrayFloat tempData = new ArrayFloat(new int[] { tLatDim0.getLength(), tLonDim0.getLength() });
            Index tempIndex = tempData.getIndex();
            for (int yPos = 0; yPos < tLatDim0.getLength(); yPos++) {
                for (int xPos = 0; xPos < tLonDim0.getLength(); xPos++) {
                    tempData.setFloat(tempIndex.set(yPos, xPos), maskData.getFloat(maskIndex.set(yPos, xPos)));
                }
            }
            WritableRaster outData = this.Resampler(tLatOriginalData, tLonOriginalData, tLonDim0.getLength(), tLatDim0.getLength(), 2, tempData, -1, T);
            for (int j = 0; j < tLatDim0.getLength(); j++) {
                for (int k = 0; k < tLonDim0.getLength(); k++) {
                    float sample = outData.getSampleFloat(k, j, 0);
                    maskMatrixT.setFloat(maskImaT.set(j, k), sample);
                }
            }
        }
        ArrayFloat maskMatrixV = new ArrayFloat.D2(latDimT.getLength(), lonDimT.getLength());
        Index maskImaV = maskMatrixV.getIndex();
        if (APPLY_MASK) {
            Variable mask = ncFileIn.findVariable("landv");
            Array maskData = mask.read();
            Index maskIndex = maskData.getIndex();
            ArrayFloat tempData = new ArrayFloat(new int[] { vLatDim0.getLength(), vLonDim0.getLength() });
            Index tempIndex = tempData.getIndex();
            for (int yPos = 0; yPos < vLatDim0.getLength(); yPos++) {
                for (int xPos = 0; xPos < vLonDim0.getLength(); xPos++) {
                    tempData.setFloat(tempIndex.set(yPos, xPos), maskData.getFloat(maskIndex.set(yPos, xPos)));
                }
            }
            WritableRaster outData = this.Resampler(vLatOriginalData, vLonOriginalData, vLonDim0.getLength(), vLatDim0.getLength(), 2, tempData, -1, V);
            for (int j = 0; j < vLatDim0.getLength(); j++) {
                for (int k = 0; k < vLonDim0.getLength(); k++) {
                    float sample = outData.getSampleFloat(k, j, 0);
                    maskMatrixV.setFloat(maskImaV.set(j, k), sample);
                }
            }
        }
        // //
        // 
        // Writing Tracers related Variables
        // 
        // //
        System.out.print("T Process complete...0.0%");
        for (int i = 0; i < NUMVARS_T; i++) {
            String varName = variablesT.get(i);
            Variable var = ncFileIn.findVariable(varName);
            Array originalVarData = var.read();
            Index varIndex = originalVarData.getIndex();
            Attribute fv = var.findAttribute("_FillValue");
            float fillValue = Float.NaN;
            if (fv != null) {
                fillValue = (fv.getNumericValue()).floatValue();
            }
            ArrayFloat tempData = new ArrayFloat(new int[] { tLatDim0.getLength(), tLonDim0.getLength() });
            Index tempIndex = tempData.getIndex();
            ArrayFloat tMatrix = new ArrayFloat.D4(timeDimT.getLength(), depthDimT.getLength(), latDimT.getLength(), lonDimT.getLength());
            Index tIma = tMatrix.getIndex();
            for (int tPos = 0; tPos < timeDim0.getLength(); tPos++) {
                for (int levelPos = 0; levelPos < depthDimT.getLength(); levelPos++) {
                    for (int yPos = 0; yPos < tLatDim0.getLength(); yPos++) {
                        for (int xPos = 0; xPos < tLonDim0.getLength(); xPos++) {
                            // Vertical Interpolation (nearest neighbor)
                            final double targetZ = new Double(this.zmaxTV[T] - (new Double(levelPos).doubleValue() * this.zperiodTV[T])).doubleValue();
                            int targetOutlev = 0;
                            double distance = Double.POSITIVE_INFINITY;
                            for (int ol = 0; ol < depthDimT.getLength(); ol++) {
                                final double designatedZ = tLevOriginalData.getDouble(tLevOriginalIndex.set(yPos, xPos, ol));
                                if (Math.abs(designatedZ - targetZ) < distance) {
                                    distance = Math.abs(designatedZ - targetZ);
                                    targetOutlev = ol;
                                }
                            }
                            tempData.setFloat(tempIndex.set(yPos, xPos), originalVarData.getFloat(varIndex.set(tPos, yPos, xPos, targetOutlev)));
                        }
                    }
                    WritableRaster outData = this.Resampler(tLatOriginalData, tLonOriginalData, tLonDim0.getLength(), tLatDim0.getLength(), 2, tempData, fillValue, T);
                    for (int j = 0; j < tLatDim0.getLength(); j++) {
                        for (int k = 0; k < tLonDim0.getLength(); k++) {
                            float sample = outData.getSampleFloat(k, j, 0);
                            if (APPLY_MASK) {
                                float maskValue = maskMatrixV.getFloat(maskImaV.set(j, k));
                                if (maskValue == 0)
                                    sample = fillValue;
                            }
                            tMatrix.setFloat(tIma.set(tPos, levelPos, j, k), sample);
                        }
                    }
                }
            }
            ncFileOut.write(varName, tMatrix);
            System.out.print("..." + new Double((new Double(i + 1).doubleValue() / new Double(NUMVARS_T).doubleValue()) * 100.0).floatValue() + "%");
        }
        // //
        // 
        // Writing Velocity related Variables
        // 
        // //
        System.out.print("V Process complete...0.0%");
        for (int i = 0; i < NUMVARS_V; i++) {
            String varName = variablesV.get(i);
            Variable var = ncFileIn.findVariable(varName);
            Array originalVarData = var.read();
            Index varIndex = originalVarData.getIndex();
            Attribute fv = var.findAttribute("_FillValue");
            float fillValue = Float.NaN;
            if (fv != null) {
                fillValue = (fv.getNumericValue()).floatValue();
            }
            ArrayFloat tempData0 = new ArrayFloat(new int[] { vLatDim0.getLength(), vLonDim0.getLength() });
            Index tempIndex0 = tempData0.getIndex();
            ArrayFloat tempData1 = new ArrayFloat(new int[] { vLatDim0.getLength(), vLonDim0.getLength() });
            Index tempIndex1 = tempData1.getIndex();
            ArrayFloat vMatrix0 = new ArrayFloat.D4(timeDimV.getLength(), depthDimV.getLength(), latDimV.getLength(), lonDimV.getLength());
            Index vIma0 = vMatrix0.getIndex();
            ArrayFloat vMatrix1 = new ArrayFloat.D4(timeDimV.getLength(), depthDimV.getLength(), latDimV.getLength(), lonDimV.getLength());
            Index vIma1 = vMatrix1.getIndex();
            for (int tPos = 0; tPos < timeDim0.getLength(); tPos++) {
                for (int levelPos = 0; levelPos < depthDimV.getLength(); levelPos++) {
                    for (int yPos = 0; yPos < vLatDim0.getLength(); yPos++) {
                        for (int xPos = 0; xPos < vLonDim0.getLength(); xPos++) {
                            // Vertical Interpolation (nearest neighbor)
                            final double targetZ = new Double(this.zmaxTV[V] - (new Double(levelPos).doubleValue() * this.zperiodTV[V])).doubleValue();
                            int targetOutlev = 0;
                            double distance = Double.POSITIVE_INFINITY;
                            for (int ol = 0; ol < depthDimV.getLength(); ol++) {
                                final double designatedZ = vLevOriginalData.getDouble(vLevOriginalIndex.set(yPos, xPos, ol));
                                if (Math.abs(designatedZ - targetZ) < distance) {
                                    distance = Math.abs(designatedZ - targetZ);
                                    targetOutlev = ol;
                                }
                            }
                            tempData0.setFloat(tempIndex0.set(yPos, xPos), originalVarData.getFloat(varIndex.set(tPos, yPos, xPos, targetOutlev, 0)));
                            tempData1.setFloat(tempIndex1.set(yPos, xPos), originalVarData.getFloat(varIndex.set(tPos, yPos, xPos, targetOutlev, 1)));
                        }
                    }
                    WritableRaster outData0 = this.Resampler(vLatOriginalData, vLonOriginalData, vLonDim0.getLength(), vLatDim0.getLength(), 2, tempData0, fillValue, V);
                    for (int j = 0; j < vLatDim0.getLength(); j++) {
                        for (int k = 0; k < vLonDim0.getLength(); k++) {
                            float sample = outData0.getSampleFloat(k, j, 0);
                            if (APPLY_MASK) {
                                float maskValue = maskMatrixV.getFloat(maskImaV.set(j, k));
                                if (maskValue == 0)
                                    sample = fillValue;
                            }
                            vMatrix0.setFloat(vIma0.set(tPos, levelPos, j, k), sample);
                        }
                    }
                    WritableRaster outData1 = this.Resampler(vLatOriginalData, vLonOriginalData, vLonDim0.getLength(), vLatDim0.getLength(), 2, tempData1, fillValue, V);
                    for (int j = 0; j < vLatDim0.getLength(); j++) {
                        for (int k = 0; k < vLonDim0.getLength(); k++) {
                            float sample = outData1.getSampleFloat(k, j, 0);
                            if (APPLY_MASK) {
                                float maskValue = maskMatrixV.getFloat(maskImaV.set(j, k));
                                if (maskValue == 0)
                                    sample = fillValue;
                            }
                            vMatrix1.setFloat(vIma1.set(tPos, levelPos, j, k), sample);
                        }
                    }
                }
            }
            ncFileOutV.write(varName + "0", vMatrix0);
            ncFileOutV.write(varName + "1", vMatrix1);
            System.out.print("..." + new Double((new Double(i + 1).doubleValue() / new Double(NUMVARS_V).doubleValue()) * 100.0).floatValue() + "%");
        }
        ncFileOut.close();
        ncFileOutV.close();
    } catch (Exception e) {
        e.printStackTrace(System.out);
    }
}

Example 15 with Index

use of com.google.datastore.admin.v1.Index in project imageio-ext by geosolutions-it.

the class NCOMConverter method run.

private void run(String fileNameIn, String fileNameOut) {
    try {
        final File fileIn = new File(fileNameIn);
        final NetcdfFile ncFileIn = NetcdfFile.open(fileNameIn);
        final File fileOut = new File(fileNameOut);
        // keep original name
        final File outputFile = File.createTempFile(fileIn.getName(), ".tmp");
        final NetcdfFileWriteable ncFileOut = NetcdfFileWriteable.createNew(outputFile.getAbsolutePath());
        boolean hasDepth = false;
        // input dimensions
        final Dimension timeDim0 = ncFileIn.findDimension("time");
        final int nTimes = timeDim0.getLength();
        final Dimension latDim0 = ncFileIn.findDimension("lat");
        final int nLat = latDim0.getLength();
        final Dimension lonDim0 = ncFileIn.findDimension("lon");
        final int nLon = lonDim0.getLength();
        // input VARIABLES
        final Variable timeOriginalVar = ncFileIn.findVariable("time");
        final Array timeOriginalData = timeOriginalVar.read();
        final Index timeOriginalIndex = timeOriginalData.getIndex();
        final DataType timeDataType = timeOriginalVar.getDataType();
        final Variable lonOriginalVar = ncFileIn.findVariable("lon");
        final DataType lonDataType = lonOriginalVar.getDataType();
        final Variable latOriginalVar = ncFileIn.findVariable("lat");
        final DataType latDataType = latOriginalVar.getDataType();
        final Array latOriginalData = latOriginalVar.read();
        final Array lonOriginalData = lonOriginalVar.read();
        // //
        // 
        // Depth related vars
        // 
        // //
        Array depthOriginalData = null;
        int nDepths = 0;
        Array depth1Data = null;
        Dimension depthDim = null;
        DataType depthDataType = null;
        // Depth
        final Variable depthOriginalVar = ncFileIn.findVariable("depth");
        if (depthOriginalVar != null) {
            nDepths = depthOriginalVar.getDimension(0).getLength();
            depthOriginalData = depthOriginalVar.read();
            depthDataType = depthOriginalVar.getDataType();
            hasDepth = true;
        }
        Dimension timeDim = ncFileOut.addDimension("time", nTimes);
        Dimension latDim = ncFileOut.addDimension(NetCDFUtilities.LAT, nLat);
        Dimension lonDim = ncFileOut.addDimension(NetCDFUtilities.LON, nLon);
        if (hasDepth)
            depthDim = ncFileOut.addDimension(NetCDFUtilities.DEPTH, nDepths);
        NetCDFConverterUtilities.copyGlobalAttributes(ncFileOut, ncFileIn.getGlobalAttributes());
        // Dimensions
        Variable timeVar = ncFileOut.addVariable("time", timeDataType, new Dimension[] { timeDim });
        NetCDFConverterUtilities.setVariableAttributes(timeOriginalVar, ncFileOut, new String[] { "long_name" });
        ncFileOut.addVariableAttribute("time", "long_name", "time");
        ncFileOut.addVariable(NetCDFUtilities.LAT, latDataType, new Dimension[] { latDim });
        NetCDFConverterUtilities.setVariableAttributes(latOriginalVar, ncFileOut);
        ncFileOut.addVariable(NetCDFUtilities.LON, lonDataType, new Dimension[] { lonDim });
        NetCDFConverterUtilities.setVariableAttributes(lonOriginalVar, ncFileOut);
        if (hasDepth) {
            ncFileOut.addVariable(NetCDFUtilities.DEPTH, depthDataType, new Dimension[] { depthDim });
            NetCDFConverterUtilities.setVariableAttributes(depthOriginalVar, ncFileOut);
        }
        // lat Variable
        Array lat1Data = NetCDFConverterUtilities.getArray(nLat, latDataType);
        NetCDFConverterUtilities.setData1D(latOriginalData, lat1Data, latDataType, nLat, true);
        // lon Variable
        Array lon1Data = NetCDFConverterUtilities.getArray(nLon, lonDataType);
        NetCDFConverterUtilities.setData1D(lonOriginalData, lon1Data, lonDataType, nLon, false);
        if (hasDepth) {
            // depth level Variable
            depth1Data = NetCDFConverterUtilities.getArray(nDepths, depthDataType);
            NetCDFConverterUtilities.setData1D(depthOriginalData, depth1Data, depthDataType, nDepths, false);
        }
        // {} Variables
        final ArrayList<String> variables = new ArrayList<String>(5);
        final HashMap<String, String> updatingValidRange = new HashMap<String, String>(5);
        int numVars = 0;
        for (int i = 0; i < NUMVARS; i++) {
            String varName = (String) VARIABLES.get(i);
            Variable var = ncFileIn.findVariable(varName);
            if (var != null) {
                variables.add(varName);
                List<Dimension> dims = var.getDimensions();
                boolean hasLocalDepth = false;
                for (Dimension dim : dims) {
                    if (dim.getName().equalsIgnoreCase("depth")) {
                        hasLocalDepth = true;
                        break;
                    }
                }
                if (hasDepth && hasLocalDepth)
                    ncFileOut.addVariable(varName, var.getDataType(), new Dimension[] { timeDim, depthDim, latDim, lonDim });
                else
                    ncFileOut.addVariable(varName, var.getDataType(), new Dimension[] { timeDim, latDim, lonDim });
                Attribute validRange = var.findAttribute(NetCDFUtilities.DatasetAttribs.VALID_MAX);
                if (validRange != null)
                    validRange = var.findAttribute(NetCDFUtilities.DatasetAttribs.VALID_MIN);
                if (validRange == null)
                    validRange = var.findAttribute(NetCDFUtilities.DatasetAttribs.VALID_RANGE);
                if (validRange == null) {
                    updatingValidRange.put(varName, "");
                    ArrayShort range = new ArrayShort(new int[] { 2 });
                    Index index = range.getIndex();
                    range.setShort(index.set(0), Short.MIN_VALUE);
                    range.setShort(index.set(1), Short.MAX_VALUE);
                    ncFileOut.addVariableAttribute(varName, NetCDFUtilities.DatasetAttribs.VALID_RANGE, range);
                }
                NetCDFConverterUtilities.setVariableAttributes(var, ncFileOut);
                numVars++;
            }
        }
        // writing bin data ...
        ncFileOut.create();
        ArrayFloat timeData = new ArrayFloat(new int[] { timeDim.getLength() });
        Index timeIndex = timeData.getIndex();
        for (int t = 0; t < timeDim.getLength(); t++) {
            timeData.setFloat(timeIndex.set(t), timeOriginalData.getFloat(timeOriginalIndex.set(t)));
        }
        ncFileOut.write("time", timeData);
        timeVar = ncFileOut.findVariable("time");
        timeDim.addCoordinateVariable(timeVar);
        ncFileOut.write(NetCDFUtilities.LAT, lat1Data);
        ncFileOut.write(NetCDFUtilities.LON, lon1Data);
        if (hasDepth) {
            Variable depthVar = ncFileOut.findVariable("depth");
            depthDim.addCoordinateVariable(depthVar);
            ncFileOut.write(NetCDFUtilities.DEPTH, depth1Data);
        }
        for (int i = 0; i < numVars; i++) {
            String varName = (String) variables.get(i);
            Variable var = ncFileIn.findVariable(varName);
            boolean hasLocalDepth = NetCDFConverterUtilities.hasThisDimension(var, "depth");
            Array originalVarArray = var.read();
            DataType varDataType = var.getDataType();
            Array destArray = null;
            int[] dimensions = null;
            if (hasDepth && hasLocalDepth) {
                dimensions = new int[] { timeDim.getLength(), depthDim.getLength(), latDim.getLength(), lonDim.getLength() };
            } else {
                dimensions = new int[] { timeDim.getLength(), latDim.getLength(), lonDim.getLength() };
            }
            destArray = NetCDFConverterUtilities.getArray(dimensions, varDataType);
            boolean findNewRange = updatingValidRange.containsKey(varName);
            final boolean setDepth = hasDepth && hasLocalDepth;
            final int[] loopLengths;
            if (setDepth)
                loopLengths = new int[] { nTimes, nDepths, nLat, nLon };
            else
                loopLengths = new int[] { nTimes, nLat, nLon };
            NetCDFConverterUtilities.writeData(ncFileOut, varName, var, originalVarArray, destArray, findNewRange, false, loopLengths, true);
        }
        ncFileOut.close();
        outputFile.renameTo(fileOut);
    } catch (Exception e) {
        // something bad happened
        if (NetCDFConverterUtilities.LOGGER.isLoggable(Level.INFO))
            NetCDFConverterUtilities.LOGGER.log(Level.INFO, e.getLocalizedMessage(), e);
        JAI.getDefaultInstance().getTileCache().flush();
    }
}