Examples with RowSequence uk.ac.starlink.table.RowSequence used on opensource projects

Example 1 with RowSequence

use of uk.ac.starlink.table.RowSequence in project gaiasky by langurmonkey.

the class STILDataProvider method loadData.

/**
 * @param ds             The data source.
 * @param factor         Length factor.
 * @param preCallback    A function that runs before.
 * @param updateCallback A function that runs after each object has loaded. Gets two longs, the first holds the current number of loaded objects and the
 *                       second holds the total number of objects to load.
 * @param postCallback   A function that runs after the data has been loaded.
 *
 * @return The list of particle records.
 */
public List<IParticleRecord> loadData(DataSource ds, double factor, Runnable preCallback, RunnableLongLong updateCallback, Runnable postCallback) {
    try {
        if (factory != null) {
            // RNG
            final Random r = new Random(123L);
            // Add extra builders
            List<TableBuilder> builders = factory.getDefaultBuilders();
            builders.add(new CsvTableBuilder());
            builders.add(new AsciiTableBuilder());
            if (preCallback != null)
                preCallback.run();
            // Try to load
            StarTable table = factory.makeStarTable(ds);
            long count = table.getRowCount();
            initLists((int) count);
            UCDParser ucdParser = new UCDParser();
            ucdParser.parse(table);
            int resampledLightCurves = 0;
            int noPeriods = 0;
            if (ucdParser.haspos) {
                BVToTeff_ballesteros bvToTEff = new BVToTeff_ballesteros();
                int nInvalidParallaxes = 0;
                long i = 0L;
                long step = Math.max(1L, Math.round(count / 100d));
                RowSequence rs = table.getRowSequence();
                while (rs.next()) {
                    Object[] row = rs.getRow();
                    try {
                        // POSITION
                        Pair<UCD, Double> a = getDoubleUcd(ucdParser.POS1, row);
                        Pair<UCD, Double> b = getDoubleUcd(ucdParser.POS2, row);
                        Pair<UCD, Double> c;
                        String unitC;
                        Pair<UCD, Double> pos3 = getDoubleUcd(ucdParser.POS3, row);
                        // Check missing pos3 -> Use default parallax
                        if (ucdParser.POS3.isEmpty() || pos3 == null || pos3.getSecond() == null || !Double.isFinite(pos3.getSecond())) {
                            c = new Pair<>(null, 0.04);
                            unitC = "mas";
                            nInvalidParallaxes++;
                        } else {
                            c = getDoubleUcd(ucdParser.POS3, row);
                            assert c != null;
                            unitC = c.getFirst().unit;
                        }
                        assert a != null;
                        assert b != null;
                        PositionType pt = ucdParser.getPositionType(a.getFirst(), b.getFirst(), c.getFirst());
                        // Check negative parallaxes -> Use default for consistency
                        if (pt.isParallax() && (c.getSecond() == null || c.getSecond().isNaN() || c.getSecond() <= 0)) {
                            c.setSecond(0.04);
                            unitC = "mas";
                            nInvalidParallaxes++;
                        }
                        Position p = new Position(a.getSecond(), a.getFirst().unit, b.getSecond(), b.getFirst().unit, c.getSecond(), unitC, pt);
                        double distPc = p.gsposition.len();
                        if ((pt.isParallax() && c.getSecond() <= 0) || !Double.isFinite(distPc) || distPc < 0) {
                            // Next
                            break;
                        }
                        p.gsposition.scl(Constants.PC_TO_U);
                        // Find out RA/DEC/Dist
                        Vector3d sph = new Vector3d();
                        Coordinates.cartesianToSpherical(p.gsposition, sph);
                        // PROPER MOTION
                        Vector3d pm;
                        double muAlphaStar = 0, muDelta = 0, radVel = 0;
                        // Only supported if position is equatorial spherical coordinates (ra/dec)
                        if (pt == PositionType.EQ_SPH_DIST || pt == PositionType.EQ_SPH_PLX) {
                            Pair<UCD, Double> pma = getDoubleUcd(ucdParser.PMRA, row);
                            Pair<UCD, Double> pmb = getDoubleUcd(ucdParser.PMDEC, row);
                            Pair<UCD, Double> pmc = getDoubleUcd(ucdParser.RADVEL, row);
                            muAlphaStar = pma != null ? pma.getSecond() : 0;
                            muDelta = pmb != null ? pmb.getSecond() : 0;
                            radVel = pmc != null ? pmc.getSecond() : 0;
                            double raRad = new Angle(a.getSecond(), a.getFirst().unit).get(AngleUnit.RAD);
                            double decRad = new Angle(b.getSecond(), b.getFirst().unit).get(AngleUnit.RAD);
                            pm = AstroUtils.properMotionsToCartesian(muAlphaStar, muDelta, radVel, raRad, decRad, distPc, new Vector3d());
                        } else {
                            pm = new Vector3d(Vector3d.Zero);
                        }
                        // MAGNITUDE
                        double appMag;
                        if (!ucdParser.MAG.isEmpty()) {
                            Pair<UCD, Double> appMagPair = getDoubleUcd(ucdParser.MAG, row);
                            assert appMagPair != null;
                            appMag = appMagPair.getSecond();
                        } else {
                            // Default magnitude
                            appMag = 15;
                        }
                        // Scale magnitude if needed
                        double magScl = (datasetOptions != null && datasetOptions.type == DatasetOptions.DatasetLoadType.STARS || (datasetOptions != null && datasetOptions.type == DatasetLoadType.VARIABLES)) ? datasetOptions.magnitudeScale : 0f;
                        appMag = appMag - magScl;
                        // Absolute magnitude to pseudo-size
                        final double absMag = AstroUtils.apparentToAbsoluteMagnitude(distPc, appMag);
                        final float size = (float) absoluteMagnitudeToPseudoSize(absMag);
                        // COLOR
                        float color;
                        if (!ucdParser.COL.isEmpty()) {
                            Pair<UCD, Double> colPair = getDoubleUcd(ucdParser.COL, row);
                            if (colPair == null) {
                                color = 0.656f;
                            } else {
                                color = colPair.getSecond().floatValue();
                            }
                        } else {
                            // Default color
                            color = 0.656f;
                        }
                        // VARIABILITY
                        float[] variMags = null;
                        double[] variTimes = null;
                        double pf = 0;
                        int nVari = 0;
                        if (ucdParser.hasvari) {
                            Pair<UCD, Double> period = getDoubleUcd(ucdParser.VARI_PERIOD, row);
                            if (!ucdParser.hasperiod || period == null || !Double.isFinite(period.getSecond())) {
                                // Skip stars without period
                                noPeriods++;
                                continue;
                            } else {
                                pf = period.getSecond();
                            }
                            Pair<UCD, double[]> variMagsPair = getDoubleArrayUcd(ucdParser.VARI_MAGS, row);
                            assert variMagsPair != null;
                            double[] variMagsDouble = variMagsPair.getSecond();
                            nVari = variMagsDouble.length;
                            variMags = new float[nVari];
                            Pair<UCD, double[]> variTimesPair = getDoubleArrayUcd(ucdParser.VARI_TIMES, row);
                            assert variTimesPair != null;
                            variTimes = variTimesPair.getSecond();
                            double[] auxMags = variMagsDouble;
                            double[] auxTimes = variTimes;
                            // SANITIZE (no NaNs)
                            List<Double> magnitudesList = new ArrayList<>();
                            List<Double> timesList = new ArrayList<>();
                            int idx = 0;
                            for (double mag : auxMags) {
                                if (Double.isFinite(mag)) {
                                    magnitudesList.add(mag - magScl);
                                    timesList.add(auxTimes[idx]);
                                }
                                idx++;
                            }
                            variMagsDouble = magnitudesList.stream().mapToDouble(Double::doubleValue).toArray();
                            variTimes = timesList.stream().mapToDouble(Double::doubleValue).toArray();
                            nVari = variMagsDouble.length;
                            // FOLD
                            List<Vector2d> list = new ArrayList<>(nVari);
                            for (int k = 0; k < nVari; k++) {
                                double phase = ((variTimes[k] - variTimes[0]) % pf);
                                list.add(new Vector2d(phase, variMagsDouble[k]));
                            }
                            list.sort(Comparator.comparingDouble(o -> o.x));
                            for (int k = 0; k < nVari; k++) {
                                Vector2d point = list.get(k);
                                variTimes[k] = point.x + variTimes[0];
                                variMagsDouble[k] = point.y;
                            }
                            // RESAMPLE (only if too many samples)
                            final int MAX_VARI = VariableGroupRenderSystem.MAX_VARI;
                            if (variMagsDouble.length > MAX_VARI) {
                                nVari = MAX_VARI;
                                double t0 = variTimes[0];
                                double tn = variTimes[variTimes.length - 1];
                                double tStep = (tn - t0) / (nVari - 1);
                                UnivariateInterpolator interp = new LinearInterpolator();
                                UnivariateFunction f = interp.interpolate(variTimes, variMagsDouble);
                                variMagsDouble = new double[nVari];
                                variTimes = new double[nVari];
                                for (idx = 0; idx < nVari; idx++) {
                                    double t = t0 + tStep * idx;
                                    variTimes[idx] = t;
                                    variMagsDouble[idx] = f.value(t);
                                }
                                resampledLightCurves++;
                            }
                            // Convert magnitudes to sizes
                            assert variMags.length == variTimes.length;
                            for (int j = 0; j < variMagsDouble.length; j++) {
                                double variAbsoluteMag = AstroUtils.apparentToAbsoluteMagnitude(distPc, variMagsDouble[j]);
                                variMags[j] = (float) absoluteMagnitudeToPseudoSize(variAbsoluteMag);
                            }
                        }
                        // EFFECTIVE TEMPERATURE
                        float tEff;
                        if (!ucdParser.TEFF.isEmpty()) {
                            Pair<UCD, Double> tEffPair = getDoubleUcd(ucdParser.TEFF, row);
                            assert tEffPair != null;
                            tEff = tEffPair.getSecond().floatValue();
                        } else {
                            // Convert B-V to T_eff using Ballesteros 2012
                            tEff = (float) bvToTEff.bvToTeff(color);
                        }
                        // RGB
                        float[] rgb = ColorUtils.BVtoRGB(color);
                        // float[] rgb = ColorUtils.teffToRGB_harre(teff);
                        float col = Color.toFloatBits(rgb[0], rgb[1], rgb[2], 1.0f);
                        // IDENTIFIER AND NAME
                        String[] names;
                        long id = -1L;
                        int hip = -1;
                        if (ucdParser.NAME.isEmpty()) {
                            // Empty name
                            if (!ucdParser.ID.isEmpty()) {
                                // We have ID
                                Pair<UCD, String> namePair = getStringUcd(ucdParser.ID, row);
                                assert namePair != null;
                                names = new String[] { namePair.getSecond() };
                                if (namePair.getFirst().colname.equalsIgnoreCase("hip")) {
                                    hip = Integer.parseInt(namePair.getSecond());
                                    id = hip;
                                } else {
                                    id = ++starId;
                                }
                            } else {
                                // Empty ID
                                id = ++starId;
                                names = new String[] { Long.toString(id) };
                            }
                        } else {
                            // We have a name
                            Pair<UCD, String>[] namePairs = getAllStringsUcd(ucdParser.NAME, row);
                            Array<String> namesArray = new Array<>(false, namePairs.length);
                            for (Pair<UCD, String> pair : namePairs) {
                                String[] currNames = pair.getSecond().split(Constants.nameSeparatorRegex);
                                for (String actualName : currNames) {
                                    if (actualName != null && !actualName.isEmpty() && !TextUtils.contains(forbiddenNameValues, actualName, true)) {
                                        namesArray.add(actualName);
                                    }
                                }
                            }
                            names = new String[namesArray.size];
                            int k = 0;
                            for (String n : namesArray) {
                                names[k++] = n;
                            }
                            if (names.length == 0) {
                                names = new String[] { Long.toString(id) };
                            }
                            // Take care of HIP stars
                            if (!ucdParser.ID.isEmpty()) {
                                Pair<UCD, String> idPair = getStringUcd(ucdParser.ID, row);
                                assert idPair != null;
                                if (idPair.getFirst().colname.equalsIgnoreCase("hip")) {
                                    hip = Integer.parseInt(idPair.getSecond());
                                    id = hip;
                                } else {
                                    id = ++starId;
                                }
                            } else {
                                id = ++starId;
                            }
                        }
                        // Populate provider lists
                        colors.put(id, rgb);
                        sphericalPositions.put(id, new double[] { sph.x, sph.y, sph.z });
                        if (datasetOptions == null || datasetOptions.type == DatasetOptions.DatasetLoadType.STARS || datasetOptions.type == DatasetOptions.DatasetLoadType.VARIABLES) {
                            double[] dataD = new double[ParticleRecord.STAR_SIZE_D];
                            float[] dataF = new float[ParticleRecord.STAR_SIZE_F];
                            dataD[ParticleRecord.I_X] = p.gsposition.x;
                            dataD[ParticleRecord.I_Y] = p.gsposition.y;
                            dataD[ParticleRecord.I_Z] = p.gsposition.z;
                            dataF[ParticleRecord.I_FPMX] = (float) pm.x;
                            dataF[ParticleRecord.I_FPMY] = (float) pm.y;
                            dataF[ParticleRecord.I_FPMZ] = (float) pm.z;
                            dataF[ParticleRecord.I_FMUALPHA] = (float) muAlphaStar;
                            dataF[ParticleRecord.I_FMUDELTA] = (float) muDelta;
                            dataF[ParticleRecord.I_FRADVEL] = (float) radVel;
                            dataF[ParticleRecord.I_FAPPMAG] = (float) appMag;
                            dataF[ParticleRecord.I_FABSMAG] = (float) absMag;
                            dataF[ParticleRecord.I_FCOL] = col;
                            dataF[ParticleRecord.I_FSIZE] = size;
                            dataF[ParticleRecord.I_FHIP] = hip;
                            // Extra
                            ObjectDoubleMap<UCD> extraAttributes = addExtraAttributes(ucdParser, row);
                            if (ucdParser.TEFF.isEmpty()) {
                                UCD tEffUCD = new UCD("phys.temperature.effective", "teff", "K", -1);
                                extraAttributes = initExtraAttributes(extraAttributes);
                                extraAttributes.put(tEffUCD, tEff);
                            } else {
                                extraAttributes = initExtraAttributes(extraAttributes);
                                extraAttributes.put(ucdParser.TEFF.first(), tEff);
                            }
                            final IParticleRecord sb;
                            if (datasetOptions != null && datasetOptions.type == DatasetLoadType.VARIABLES || variMags != null) {
                                sb = new VariableRecord(dataD, dataF, nVari, pf, variMags, variTimes, id, names, extraAttributes);
                            } else {
                                sb = new ParticleRecord(dataD, dataF, id, names, extraAttributes);
                            }
                            list.add(sb);
                            int appMagClamp = (int) MathUtilsd.clamp(appMag, 0, 21);
                            countsPerMag[appMagClamp] += 1;
                        } else if (datasetOptions.type == DatasetOptions.DatasetLoadType.PARTICLES) {
                            double[] point = new double[3];
                            point[ParticleRecord.I_X] = p.gsposition.x;
                            point[ParticleRecord.I_Y] = p.gsposition.y;
                            point[ParticleRecord.I_Z] = p.gsposition.z;
                            // Extra
                            ObjectDoubleMap<UCD> extraAttributes = addExtraAttributes(ucdParser, row);
                            IParticleRecord pb = new ParticleRecord(point, null, null, names, extraAttributes);
                            list.add(pb);
                        }
                    } catch (Exception e) {
                        logger.debug(e);
                        logger.debug("Exception parsing row " + i + ": skipping");
                    }
                    i++;
                    if (updateCallback != null && i % step == 0) {
                        updateCallback.run(i, count);
                    }
                }
                if (nInvalidParallaxes > 0) {
                    logger.warn("Found " + nInvalidParallaxes + " rows with nonexistent or negative parallax. Using the default 0.04 mas for them.");
                }
                if (resampledLightCurves > 0) {
                    logger.warn(resampledLightCurves + " light curves resampled to fit in default array size (=" + VariableGroupRenderSystem.MAX_VARI + ")");
                }
                if (noPeriods > 0) {
                    logger.warn("Skipped " + noPeriods + " variable stars without a period");
                }
            } else {
                logger.error("Table not loaded: Position not found");
            }
        }
    } catch (Exception e) {
        logger.error(e);
    } finally {
        if (postCallback != null)
            postCallback.run();
    }
    return list;
}

Example 2 with RowSequence

use of uk.ac.starlink.table.RowSequence in project gaiasky by langurmonkey.

the class StarClusterLoader method loadClustersStil.

/**
 * Loads clusters from a STIL data source.
 * @param ds The data source.
 * @param clusters The clusters list.
 * @throws IOException
 */
private void loadClustersStil(DataSource ds, Array<StarCluster> clusters) throws IOException {
    // Add extra builders
    StarTableFactory factory = new StarTableFactory();
    List builders = factory.getDefaultBuilders();
    builders.add(new CsvTableBuilder());
    builders.add(new AsciiTableBuilder());
    // Try to load
    StarTable table = factory.makeStarTable(ds);
    Map<ClusterProperties, Integer> indices = parseHeader(table);
    if (!checkIndices(indices)) {
        logger.error("At least 'ra', 'dec', 'pllx'|'dist', 'radius' and 'name' are needed, please check your columns!");
        return;
    }
    RowSequence rs = table.getRowSequence();
    while (rs.next()) {
        Object[] row = rs.getRow();
        String[] names = parseName(row[indices.get(ClusterProperties.NAME)].toString());
        double ra = getDouble(row, ClusterProperties.RA, indices, table, "deg");
        double rarad = Math.toRadians(ra);
        double dec = getDouble(row, ClusterProperties.DEC, indices, table, "deg");
        double decrad = Math.toRadians(dec);
        double distpc = 0;
        if (indices.containsKey(ClusterProperties.DIST)) {
            distpc = getDouble(row, ClusterProperties.DIST, indices, table, "pc");
        } else if (indices.containsKey(ClusterProperties.PLLX)) {
            distpc = 1000d / getDouble(row, ClusterProperties.PLLX, indices, table, "mas");
        }
        double dist = distpc * Constants.PC_TO_U;
        double mualphastar = getDouble(row, ClusterProperties.PMRA, indices, table, "mas/yr");
        double mudelta = getDouble(row, ClusterProperties.PMDE, indices, table, "mas/yr");
        double radvel = getDouble(row, ClusterProperties.RV, indices, table, "km/s");
        double radius = getDouble(row, ClusterProperties.RADIUS, indices, table, "deg");
        int nstars = getInteger(row, ClusterProperties.NSTARS, indices);
        addCluster(names, ra, rarad, dec, decrad, dist, distpc, mualphastar, mudelta, radvel, radius, nstars, clusters);
    }
    for (StarCluster c : clusters) {
        c.initialize();
    }
}