Examples with Variable org.flyte.api.v1.Variable used on opensource projects

Example 41 with Variable

use of org.flyte.api.v1.Variable in project risky by amsa-code.

the class DistanceTravelledCalculator method saveCalculationResultAsNetcdf.

public static void saveCalculationResultAsNetcdf(Options options, CalculationResult calculationResult, String filename) {
    List<CellValue> list = calculationResult.getCells().toList().toBlocking().single();
    int maxLonIndex = list.stream().map(cell -> options.getGrid().cellAt(cell.getCentreLat(), cell.getCentreLon())).filter(// 
    x -> x.isPresent()).map(x -> x.get().getLonIndex()).max(// 
    Comparator.<Long>naturalOrder()).get().intValue();
    int maxLatIndex = list.stream().map(cell -> options.getGrid().cellAt(cell.getCentreLat(), cell.getCentreLon())).filter(x -> x.isPresent()).map(x -> x.get().getLatIndex()).max(Comparator.<Long>naturalOrder()).get().intValue();
    File file = new File(filename);
    // Create the file.
    NetcdfFileWriter f = null;
    try {
        // Create new netcdf-3 file with the given filename
        f = NetcdfFileWriter.createNew(NetcdfFileWriter.Version.netcdf3, file.getPath());
        // In addition to the latitude and longitude dimensions, we will
        // also create latitude and longitude netCDF variables which will
        // hold the actual latitudes and longitudes. Since they hold data
        // about the coordinate system, the netCDF term for these is:
        // "coordinate variables."
        Dimension dimLat = f.addDimension(null, "latitude", maxLatIndex + 1);
        Dimension dimLon = f.addDimension(null, "longitude", maxLonIndex + 1);
        List<Dimension> dims = new ArrayList<Dimension>();
        dims.add(dimLat);
        dims.add(dimLon);
        // coordinate variables
        Variable vLat = f.addVariable(null, "latitude", DataType.DOUBLE, "latitude");
        Variable vLon = f.addVariable(null, "longitude", DataType.DOUBLE, "longitude");
        // value variables
        Variable vDensity = f.addVariable(null, "traffic_density", DataType.DOUBLE, dims);
        // Define units attributes for coordinate vars. This attaches a
        // text attribute to each of the coordinate variables, containing
        // the units.
        vLon.addAttribute(new Attribute("units", "degrees_east"));
        vLat.addAttribute(new Attribute("units", "degrees_north"));
        // Define units attributes for variables.
        vDensity.addAttribute(new Attribute("units", "nm-1"));
        vDensity.addAttribute(new Attribute("long_name", ""));
        // Write the coordinate variable data. This will put the latitudes
        // and longitudes of our data grid into the netCDF file.
        f.create();
        {
            Array dataLat = Array.factory(DataType.DOUBLE, new int[] { dimLat.getLength() });
            Array dataLon = Array.factory(DataType.DOUBLE, new int[] { dimLon.getLength() });
            // set latitudes
            for (int i = 0; i <= maxLatIndex; i++) {
                dataLat.setDouble(i, options.getGrid().centreLat(i));
            }
            // set longitudes
            for (int i = 0; i <= maxLonIndex; i++) {
                dataLon.setDouble(i, options.getGrid().centreLon(i));
            }
            f.write(vLat, dataLat);
            f.write(vLon, dataLon);
        }
        // write the value variable data
        {
            int[] iDim = new int[] { dimLat.getLength(), dimLon.getLength() };
            Array dataDensity = ArrayDouble.D2.factory(DataType.DOUBLE, iDim);
            Index2D idx = new Index2D(iDim);
            for (CellValue point : list) {
                Optional<Cell> cell = options.getGrid().cellAt(point.getCentreLat(), point.getCentreLon());
                if (cell.isPresent()) {
                    idx.set((int) cell.get().getLatIndex(), (int) cell.get().getLonIndex());
                    dataDensity.setDouble(idx, point.getValue());
                }
            }
            f.write(vDensity, dataDensity);
        }
    } catch (IOException | InvalidRangeException e) {
        throw new RuntimeException(e);
    } finally {
        if (f != null) {
            try {
                f.close();
            } catch (IOException ioe) {
                throw new RuntimeException(ioe);
            }
        }
    }
}

Example 42 with Variable

use of org.flyte.api.v1.Variable in project risky by amsa-code.

the class NetCdfWriter method addVariable.

public <T> Var<T> addVariable(String shortName, Optional<String> longName, Optional<String> units, Optional<String> encoding, Class<T> cls, int numRecords) {
    Preconditions.checkNotNull(shortName);
    Preconditions.checkNotNull(longName);
    Preconditions.checkNotNull(units);
    Preconditions.checkNotNull(encoding);
    Preconditions.checkNotNull(cls);
    Dimension dimension = f.addDimension(null, shortName, numRecords);
    Variable variable = f.addVariable(null, shortName, toDataType(cls), Arrays.asList(dimension));
    if (longName.isPresent())
        variable.addAttribute(new Attribute("long_name", longName.get()));
    if (units.isPresent())
        variable.addAttribute(new Attribute("units", units.get()));
    if (encoding.isPresent())
        variable.addAttribute(new Attribute("encoding", encoding.get()));
    return new Var<T>(this, variable, cls);
}

Example 43 with Variable

use of org.flyte.api.v1.Variable in project anyline by anylineorg.

the class NCUtil method getVariableNames.

/**
 * 变量名称列表
 * @return return
 */
public List<String> getVariableNames() {
    List<String> list = new ArrayList<>();
    List<Variable> variables = getVariables();
    for (Variable var : variables) {
        list.add(var.getFullName());
    }
    return list;
}

Example 44 with Variable

use of org.flyte.api.v1.Variable in project gridfour by gwlucastrig.

the class ExtractData method process.

void process(PrintStream ps, String product, String inputPath) throws IOException, InvalidRangeException {
    // Open the NetCDF file -----------------------------------
    ps.println("Reading data from " + inputPath);
    NetcdfFile ncfile = NetcdfFile.open(inputPath);
    // Inspect the content of the file ----------------------------
    // Start with the high-level metadata elements that describe the
    // entire file
    ps.println("NetCDF File Type: " + ncfile.getFileTypeDescription());
    ps.println("");
    ps.println("Global attributes attached to file---------------------------");
    List<Attribute> attributes = ncfile.getGlobalAttributes();
    for (Attribute a : attributes) {
        ps.println(a.toString());
    }
    // The content of NetCDF files is accessed through the use of the
    // NetCDF class named Variable. Get all Variables defined in the
    // current file and print a summary of their metadata to the text output.
    // The Java NetCDF team implemented good "toString()" methods
    // for the variable class.  So printing their metadata is quite easy.
    ps.println("");
    ps.println("Variables found in file--------------------------------------");
    List<Variable> variables = ncfile.getVariables();
    for (Variable v : variables) {
        ps.println("\n" + v.toString());
    }
    // Identify which Variable instances carry information about the
    // geographic (latitude/longitude) coordinate system and also which
    // carry information for elevation and bathymetry.
    // In NetCDF, Variable instances area associated
    // with arbitrary "name" attributes assigned to the objects
    // when the data product is created. We can pull these variables
    // out of the NetCDF file using their names.  However,
    // ETOPO1 and GEBCO_2019 use different Variable names to identify
    // their content.
    // the Variable that carries row-latitude information
    Variable lat;
    // the Variable that carries column-longitude information
    Variable lon;
    // the variable that carries elevation and bathymetry
    Variable z;
    // will be set to either "ETOPO1" or "GEBCO"
    String label;
    float[][] palette;
    int reportingCount;
    if (product.startsWith("ETOP")) {
        label = "ETOPO1";
        lat = ncfile.findVariable("y");
        lon = ncfile.findVariable("x");
        z = ncfile.findVariable("z");
        palette = ExamplePalettes.paletteETOPO1;
        reportingCount = 1000;
    } else {
        // the product is GEBCO
        label = "GEBCO";
        lat = ncfile.findVariable("lat");
        lon = ncfile.findVariable("lon");
        z = ncfile.findVariable("elevation");
        palette = ExamplePalettes.paletteGEBCO;
        reportingCount = 10000;
    }
    // using the variables from above, extract coordinate system
    // information for the product and print it to the output.
    // we will use this data below for computing the volume of the
    // world's oceans.
    ExtractionCoordinates coords = new ExtractionCoordinates(lat, lon);
    coords.summarizeCoordinates(ps);
    // Get the dimensions of the raster (grid) elevation/bathymetry data.
    // 
    // NetCDF uses an concept named "rank" to define the rank of a
    // Variable.  A variable of rank 1 is essentially a vector (or one
    // dimensional array).  A variable of rank 2 has rows and columns,
    // and is essentially a matrix.  Higher rank variables are not uncommon.
    // 
    // In ETOPO1 and GEBCO_2019, the elevation/bathymetry data is given
    // as a raster (grid), so the rank will be two (for rows and columns).
    // NetCDF data is always given in row-major order.
    // 
    // NetCDF Variables also use the concept of "shape". In this case,
    // the "shape" element tells you the number of rows and columns
    // in the elevation variable.
    // The getShape() method returns an array of integers dimensioned to
    // the rank.  Since the "z" variable is a raster, the return from
    // getShape() will be an array of two values where shape[0] is
    // the number of rows in the product and shape[1] is the number of
    // columns.
    // ETOPO1:        10800 rows and 21600 columns.
    // GEBCO 2019:    43200 rows and 86400 columns
    // In both these products, the rows are arranged from south-to-north,
    // starting near the South Pole (-90 degrees) and ending near
    // the North Pole (90 degrees).
    // ETOPO1 has a uniform point spacing a 1-minute of arc.
    // GEBCO has a uniform point spacing of 15-seconds of arc.
    // Thus there are 4 times as many rows and 4 times as many columns
    // in GEBCO versus ETOPO1.
    int rank = z.getRank();
    int[] shape = z.getShape();
    int nRows = shape[0];
    int nCols = shape[1];
    ps.format("Rows:      %8d%n", nRows);
    ps.format("Columns:   %8d%n", nCols);
    // The output for this application is an image that is
    // 720 pixels wide and 360 pixels high.  Since the resulution of
    // the two products is much higher than that, we need to compute a
    // pixel scale value for down-sampling the source data.
    // As we read the data, we will combine blocks of elevation/bathymetry
    // values into a average value.
    int imageHeight = 360;
    int imageWidth = 720;
    int pixelScale = nCols / imageWidth;
    double[] sampleSum = new double[imageHeight * imageWidth];
    ps.format("Down scaling data %d to 1 for image (%d values per pixel)%n", pixelScale, pixelScale * pixelScale);
    // we also wish to collect the minimum and maximum values of the data.
    double zMin = Double.POSITIVE_INFINITY;
    double zMax = Double.NEGATIVE_INFINITY;
    // naturally, the source data products contain far too many data values
    // for us to read into memory all at once.  We could read them one-at-a-time,
    // but that would entail a lot of overhead and would slow processing
    // considerably.  So we read the data one-row-at-a-time.
    // That pattern is not always to best for some products, but it works
    // quite well for both ETOPO1 and GEBCO 2019.
    // The readOrigin variable allows the application to specify where
    // it wants to read the data from. The readShape variable tells
    // NetCDF how many rows and columns to read.
    int[] readOrigin = new int[rank];
    int[] readShape = new int[rank];
    // Finally, we are going to use the input data to estimate both the
    // surface area and volume of the world's oceans.  This calculation
    // is not authoritative, and is performed here only to illustrate
    // a potential use of the data.  We perform the calculation by
    // obtaining the surface area for each "cell" in the input raster
    // (surface area per cell will vary by latitude).  Then, if the sample
    // is less than zero, we treat it as an ocean depth and add the
    // computed area and volume to a running sum.
    double areaSum = 0;
    double volumeSum = 0;
    double depthSum = 0;
    long nDepth = 0;
    long time0 = System.nanoTime();
    for (int iRow = 0; iRow < nRows; iRow++) {
        int imageRow = imageHeight - 1 - iRow / pixelScale;
        double areaOfCellsInRow = coords.getAreaOfEachCellInRow(iRow);
        if (iRow % reportingCount == 0) {
            System.out.println("Processing row " + iRow);
        }
        int row0 = iRow;
        int col0 = 0;
        readOrigin[0] = row0;
        readOrigin[1] = col0;
        readShape[0] = 1;
        readShape[1] = nCols;
        Array array = z.read(readOrigin, readShape);
        for (int iCol = 0; iCol < nCols; iCol++) {
            int imageCol = iCol / pixelScale;
            int index = imageRow * imageWidth + imageCol;
            double sample = array.getDouble(iCol);
            if (sample <= -32767) {
                // neither ETOPO1 or GEBCO contain "no-data" points.  However,
                // there are some similar products that do.  Treat these
                // as not-a-number
                sample = Float.NaN;
            }
            if (sample < zMin) {
                zMin = sample;
            }
            if (sample > zMax) {
                zMax = sample;
            }
            sampleSum[index] += sample;
            if (sample < 0) {
                // it's water
                areaSum += areaOfCellsInRow;
                volumeSum -= areaOfCellsInRow * sample / 1000.0;
                depthSum -= sample;
                nDepth++;
            }
        }
    }
    ncfile.close();
    long time1 = System.nanoTime();
    double deltaT = (time1 - time0) / 1.0e+9;
    ps.format("Time to read input file %7.3f second%n", deltaT);
    ps.format("Min Value:              %7.3f meters%n", zMin);
    ps.format("Max Value:              %7.3f meters%n", zMax);
    ps.format("Surface area of oceans %20.1f km^2%n", areaSum);
    ps.format("Volume of oceans       %20.1f km^3%n", volumeSum);
    ps.format("Mean depth of oceans   %20.1f m%n", depthSum / nDepth);
    ps.flush();
    int[] argb = new int[imageHeight * imageWidth];
    float[] zPixel = new float[sampleSum.length];
    double nCellsPerPixel = pixelScale * pixelScale;
    for (int i = 0; i < sampleSum.length; i++) {
        zPixel[i] = (float) (sampleSum[i] / nCellsPerPixel);
        argb[i] = getRgb(palette, zPixel[i]);
    }
    String outputPath = "Test" + label + ".png";
    File outputImage = new File(outputPath);
    BufferedImage bImage = new BufferedImage(imageWidth, imageHeight, BufferedImage.TYPE_INT_ARGB);
    bImage.setRGB(0, 0, imageWidth, imageHeight, argb, 0, imageWidth);
    Graphics graphics = bImage.getGraphics();
    graphics.setColor(Color.darkGray);
    graphics.drawRect(0, 0, imageWidth - 1, imageHeight - 1);
    ImageIO.write(bImage, "PNG", outputImage);
    // Shaded Relief demo ------------------------------------------------
    // Create a shaded-relief demo by using a simple illumination model
    // and applying Gridfour's B-Spline interpolator to compute the
    // surface normal.  The x and y scales that are needed by the interpolator
    // are derived using the meters-per-pixel value which is computed from
    // the radius of the earth and the number of pixels in the image.
    // Because the distance across one degree of longitude decreases
    // as the magnitude of the latitude increases, an adjustment is
    // made to the xScale factor.  Finally, a steeping factor is added to
    // make the shading a bit more pronounced in the image.
    // 
    // Specify the parameters for the illumination source (the "sun")
    double ambient = 0.2;
    double steepen = 50.0;
    double sunAzimuth = Math.toRadians(145);
    double sunElevation = Math.toRadians(60);
    // create a unit vector pointing at illumination source
    double cosA = Math.cos(sunAzimuth);
    double sinA = Math.sin(sunAzimuth);
    double cosE = Math.cos(sunElevation);
    double sinE = Math.sin(sunElevation);
    double xSun = cosA * cosE;
    double ySun = sinA * cosE;
    double zSun = sinE;
    double yScale = Math.PI * earthRadiusM / imageHeight;
    InterpolatorBSpline bSpline = new InterpolatorBSpline();
    InterpolationResult result = new InterpolationResult();
    for (int iRow = 0; iRow < imageHeight; iRow++) {
        double yRow = iRow + 0.5;
        double rowLatitude = 90 - 180.0 * yRow / imageHeight;
        double xScale = yScale * Math.cos(Math.toRadians(rowLatitude));
        for (int iCol = 0; iCol < imageWidth; iCol++) {
            double xCol = iCol + 0.5;
            bSpline.interpolate(yRow, xCol, imageHeight, imageWidth, zPixel, yScale, xScale, InterpolationTarget.FirstDerivatives, result);
            // double z = result.z;  not used, included for documentation
            double nx = -result.zx * steepen;
            double ny = result.zy * steepen;
            double s = Math.sqrt(nx * nx + ny * ny + 1);
            nx /= s;
            ny /= s;
            double nz = 1 / s;
            double dot = nx * xSun + ny * ySun + nz * zSun;
            double shade = ambient;
            if (dot > 0) {
                shade = dot * (1 - ambient) + ambient;
            }
            int index = iRow * imageWidth + iCol;
            argb[index] = getRgb(palette, zPixel[index], shade);
        }
    }
    outputPath = "ShadedRelief_" + label + ".png";
    outputImage = new File(outputPath);
    bImage = new BufferedImage(imageWidth, imageHeight, BufferedImage.TYPE_INT_ARGB);
    bImage.setRGB(0, 0, imageWidth, imageHeight, argb, 0, imageWidth);
    graphics = bImage.getGraphics();
    graphics.setColor(Color.darkGray);
    graphics.drawRect(0, 0, imageWidth - 1, imageHeight - 1);
    ImageIO.write(bImage, "PNG", outputImage);
}

Example 45 with Variable

use of org.flyte.api.v1.Variable in project flytekit-java by flyteorg.

the class ProtoUtilTest method shouldSerDeTaskTemplate.

@Test
void shouldSerDeTaskTemplate() {
    Container container = Container.builder().command(ImmutableList.of("echo")).args(ImmutableList.of("hello world")).env(ImmutableList.of(KeyValuePair.of("key", "value"))).image("alpine:3.7").build();
    Variable stringVar = ApiUtils.createVar(SimpleType.STRING);
    Variable integerVar = ApiUtils.createVar(SimpleType.INTEGER);
    TypedInterface interface_ = TypedInterface.builder().inputs(ImmutableMap.of("x", stringVar)).outputs(ImmutableMap.of("y", integerVar)).build();
    RetryStrategy retries = RetryStrategy.builder().retries(4).build();
    TaskTemplate template = TaskTemplate.builder().container(container).type("custom-task").interface_(interface_).retries(retries).custom(Struct.of(ImmutableMap.of("custom-prop", Struct.Value.ofStringValue("custom-value")))).build();
    Tasks.TaskTemplate templateProto = Tasks.TaskTemplate.newBuilder().setContainer(Tasks.Container.newBuilder().setImage("alpine:3.7").addCommand("echo").addArgs("hello world").addEnv(Literals.KeyValuePair.newBuilder().setKey("key").setValue("value").build()).build()).setMetadata(Tasks.TaskMetadata.newBuilder().setRuntime(Tasks.RuntimeMetadata.newBuilder().setType(Tasks.RuntimeMetadata.RuntimeType.FLYTE_SDK).setFlavor(ProtoUtil.RUNTIME_FLAVOR).setVersion(ProtoUtil.RUNTIME_VERSION).build()).setRetries(Literals.RetryStrategy.newBuilder().setRetries(4).build()).build()).setInterface(Interface.TypedInterface.newBuilder().setInputs(Interface.VariableMap.newBuilder().putVariables("x", Interface.Variable.newBuilder().setType(Types.LiteralType.newBuilder().setSimple(Types.SimpleType.STRING).build()).build()).build()).setOutputs(Interface.VariableMap.newBuilder().putVariables("y", Interface.Variable.newBuilder().setType(Types.LiteralType.newBuilder().setSimple(Types.SimpleType.INTEGER).build()).build()).build()).build()).setType("custom-task").setCustom(com.google.protobuf.Struct.newBuilder().putFields("custom-prop", Value.newBuilder().setStringValue("custom-value").build()).build()).build();
    Tasks.TaskTemplate serializedTemplate = ProtoUtil.serialize(template);
    TaskTemplate deserializedTemplate = ProtoUtil.deserialize(templateProto);
    assertThat(serializedTemplate, equalTo(templateProto));
    assertThat(deserializedTemplate, equalTo(template));
}