Examples with Fix au.gov.amsa.risky.format.Fix used on opensource projects

Example 26 with Fix

use of au.gov.amsa.risky.format.Fix in project risky by amsa-code.

the class StreamsTest method testExtractFixFromAisPositionB.

@Test
public void testExtractFixFromAisPositionB() throws IOException {
    InputStream is = new ByteArrayInputStream("\\s:MSQ - Mt Cootha,c:1421877742*76\\!AIVDM,1,1,,B,B7P?oe00FRg9t`L4T4IV;wbToP06,0*2F".getBytes(Charset.forName("UTF-8")));
    Fix fix = Streams.extractFixes(Streams.nmeaFrom(is)).toBlocking().single();
    assertEquals(503576500, fix.mmsi());
    assertEquals(-27.46356391906, fix.lat(), PRECISION);
    assertEquals(1421877742000L, fix.time());
    assertEquals(AisClass.B, fix.aisClass());
    assertEquals(BinaryFixes.SOURCE_PRESENT_BUT_UNKNOWN, (int) fix.source().get());
    assertFalse(fix.navigationalStatus().isPresent());
    assertFalse(fix.latencySeconds().isPresent());
    System.out.println(fix);
    is.close();
}

Example 27 with Fix

use of au.gov.amsa.risky.format.Fix in project risky by amsa-code.

the class ModelEmpty method main.

public static void main(String[] args) {
    File file = new File("/media/an/binary-fixes-5-minute/2014/565187000.track");
    Observable<Fix> source = BinaryFixes.from(file, true);
    source.subscribe(System.out::println);
}

Example 28 with Fix

use of au.gov.amsa.risky.format.Fix in project risky by amsa-code.

the class VesselsInCoverageMain method main.

public static void main(String[] args) throws IOException {
    long t = System.currentTimeMillis();
    File out = new File("target/mmsi.txt");
    out.delete();
    try (BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(out)))) {
        Pattern pattern = Pattern.compile(".*\\.track");
        List<File> list = new ArrayList<File>();
        list.addAll(Files.find(new File("/media/an/binary-fixes-5-minute/2014"), pattern));
        list.addAll(Files.find(new File("/media/an/binary-fixes-5-minute/2015"), pattern));
        list.addAll(Files.find(new File("/media/an/binary-fixes-5-minute/2016"), pattern));
        AtomicInteger count = new AtomicInteger();
        // 
        Observable.from(list).groupBy(f -> count.getAndIncrement() % // 
        Runtime.getRuntime().availableProcessors()).flatMap(// 
        files -> toFixes(files, Schedulers.computation())).distinct(// 
        fix -> fix.mmsi() + fix.aisClass().name()).sorted(// 
        (a, b) -> Integer.compare(a.mmsi(), b.mmsi())).filter(// 
        fix -> MmsiValidator2.INSTANCE.isValid((long) fix.mmsi())).doOnNext(// 
        fix -> write(writer, fix)).toBlocking().subscribe();
    }
    System.out.println((System.currentTimeMillis() - t) + "ms");
}

Example 29 with Fix

use of au.gov.amsa.risky.format.Fix in project risky by amsa-code.

the class AdhocMain2 method main.

public static void main(String[] args) throws IOException {
    String folder = "/home/dave/Downloads";
    // String folder = "/media/an/temp";
    File file = new File(folder + "/2017-11-16.fix");
    int choice = 2;
    if (choice == 1) {
        // write the file
        try (OutputStream os = new BufferedOutputStream(new FileOutputStream(file, true))) {
            AtomicLong count = new AtomicLong();
            // 
            Streams.nmeaFromGzip(folder + "/2017-11-16.txt.gz").compose(// 
            x -> Streams.extractFixes(x)).doOnNext(// 
            log(count)).forEach(// 
            f -> BinaryFixes.write(f, os, BinaryFixesFormat.WITH_MMSI));
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
    } else if (choice == 2) {
        // print out histogram (which turns out to be uniform surprisingly)
        // and files
        long minTime = // 
        BinaryFixes.from(file, false, BinaryFixesFormat.WITH_MMSI).map(x -> x.time()).reduce((x, y) -> Math.min(x, y)).toBlocking().single();
        long maxTime = // 
        BinaryFixes.from(file, false, BinaryFixesFormat.WITH_MMSI).map(x -> x.time()).reduce((x, y) -> Math.max(x, y)).toBlocking().single();
        System.out.println("start=" + new Date(minTime) + ", finish=" + new Date(maxTime) + ", [" + minTime + "," + maxTime + "]");
        int bits = 10;
        int dimensions = 3;
        SmallHilbertCurve h = HilbertCurve.small().bits(bits).dimensions(dimensions);
        long maxIndexes = 1L << bits * dimensions;
        long maxOrdinates = 1L << bits;
        // get ranges for Sydney query to measure effectiveness
        float lat1 = -33.806477f;
        float lon1 = 151.181767f;
        long t1 = minTime + (maxTime - minTime) / 2;
        float lat2 = -33.882896f;
        float lon2 = 151.281330f;
        long t2 = t1 + TimeUnit.MINUTES.toMillis(60);
        int splits = 4;
        System.out.println("calculating ranges");
        long timer = System.currentTimeMillis();
        Ranges ranges = h.query(scalePoint(lat1, lon1, t1, minTime, maxTime, maxOrdinates), scalePoint(lat2, lon2, t2, minTime, maxTime, maxOrdinates), splits);
        System.out.println("ranges calculated in " + (System.currentTimeMillis() - timer) + "ms");
        ranges.forEach(System.out::println);
        System.out.println("numRanges=" + ranges.size());
        int numPartitions = 10;
        int[] counts = new int[numPartitions];
        long step = maxIndexes / numPartitions;
        AtomicLong inBounds = new AtomicLong();
        AtomicLong inRanges = new AtomicLong();
        AtomicLong missed = new AtomicLong();
        AtomicLong count = new AtomicLong();
        // 
        BinaryFixes.from(file, false, BinaryFixesFormat.WITH_MMSI).doOnNext(// 
        log(count)).forEach(fix -> {
            long index = h.index(scalePoint(fix.lat(), fix.lon(), fix.time(), minTime, maxTime, maxOrdinates));
            int partition = (int) (index / step);
            counts[partition]++;
            boolean inRange = false;
            for (Range r : ranges) {
                if (index >= r.low() && index <= r.high()) {
                    inRanges.incrementAndGet();
                    inRange = true;
                    break;
                }
            }
            if (between(Math.min(lat1, lat2), fix.lat(), Math.max(lat1, lat2)) && // 
            between(Math.min(lon1, lon2), fix.lon(), Math.max(lon1, lon2)) && fix.time() >= t1 && fix.time() <= t2) {
                inBounds.incrementAndGet();
                if (!inRange) {
                    missed.incrementAndGet();
                }
            }
        });
        System.out.println("ranges=" + ranges.size() + ", hit rate=" + inBounds.get() / ((double) inRanges.get()) + ", missed=" + missed.get() + " of " + inBounds.get());
        System.out.println();
        System.out.println("===============");
        System.out.println("== HISTOGRAM ==");
        System.out.println("===============");
        long sum = 0;
        for (int i = 0; i < numPartitions; i++) {
            sum += counts[i];
        }
        DecimalFormat df = new DecimalFormat("0.00000");
        for (int i = 0; i < numPartitions; i++) {
            if (counts[i] != 0) {
                System.out.println(i + " -> " + df.format(counts[i] * 100.0 / sum));
            }
        }
        System.out.println("total=" + sum);
        System.out.println();
        System.exit(0);
        System.out.println("===================");
        System.out.println("== WRITING FILES ==");
        System.out.println("===================");
        // need to write out sorted index and record in one binary file
        // and write an index file of reasonable size that can be quickly downloaded and
        // cached by searcher.
        // index file will have a column of index and positions. index will be a long,
        // position also a long so 16 bytes per entry
        // go for a 100K index file as first guess. So that is 100,000/16 = 6250 entries
        // that should be equally spaced in terms of number of
        // records between them. With 20m records in the file that is 3200 records per
        // index entry. If 20 ranges are read that could mean 20x3200 records = 2.2MB
        // read
        // unnecessarily
        OutputStream[] outs = new OutputStream[numPartitions];
        for (int i = 0; i < outs.length; i++) {
            outs[i] = new BufferedOutputStream(new FileOutputStream(new File(folder + "/partition" + i + ".fix")));
        }
        count.set(0);
        ByteBuffer bb = BinaryFixes.createFixByteBuffer(BinaryFixesFormat.WITH_MMSI);
        // 
        BinaryFixes.from(file, false, BinaryFixesFormat.WITH_MMSI).doOnNext(// 
        log(count)).doOnNext(fix -> {
            long x = Math.round(Math.floor((fix.lat() + 90) / 180.0 * maxIndexes));
            long y = Math.round(Math.floor((fix.lon() + 180) / 360.0 * maxIndexes));
            long z = Math.round(Math.floor((fix.time() - minTime) / ((double) maxTime - minTime) * maxIndexes));
            long index = h.index(x, y, z);
            int partition = (int) (index / step);
            bb.clear();
            BinaryFixes.write(fix, bb, BinaryFixesFormat.WITH_MMSI);
            try {
                outs[partition].write(bb.array());
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
        }).doOnError(// 
        t -> t.printStackTrace()).subscribe();
        for (int i = 0; i < outs.length; i++) {
            outs[i].close();
        }
    }
}

Example 30 with Fix

use of au.gov.amsa.risky.format.Fix in project risky by amsa-code.

the class LMSAdhocMain method main.

public static void main(String[] args) throws FileNotFoundException, IOException {
    Map<Integer, Info> ships = ShipStaticData.getMapFromResource("/ship-data-2014.txt");
    Pattern pattern = Pattern.compile(".*\\.track");
    File base = new File("/media/an/binary-fixes-lms2");
    List<File> files = Files.find(new File(base, "2016"), pattern);
    log.info("files=" + files.size());
    Func2<Fix, Fix, Integer> ascendingTime = (a, b) -> ((Long) a.time()).compareTo(b.time());
    String[] shapes = new String[] { "fremantle_port_limits_pl.zip", "dampier_port_limits_pl.zip", "port_hedland_port_limits_pl.zip" };
    File shapeBase = new File("/media/an/shapefiles/port-boundaries");
    DateTimeFormatter formatter = DateTimeFormatter.ofPattern("dd/MM/yyyy HH:mm").withZone(ZoneOffset.UTC);
    System.out.println("region, mmsi, imo, class, type, lat, lon, timeUTC");
    for (String shape : shapes) {
        final String shapeName = shape.replace("_port_limits_pl.zip", "");
        try (InputStream is = new FileInputStream(new File(shapeBase, shape))) {
            Shapefile region = Shapefile.fromZip(is);
            Observable.from(files).filter(file -> !file.getName().startsWith("503")).concatMap(file -> detectCrossings(file, region)).toSortedList(// 
            ascendingTime).flatMapIterable(// 
            Functions.identity()).doOnNext(fix -> {
                Info info = ships.get(fix.mmsi());
                String type = "";
                if (info != null) {
                    if (info.shipType.isPresent())
                        type = ShipTypeDecoder.getShipType(info.shipType.get());
                    else
                        type = "";
                }
                String t = formatter.format(ZonedDateTime.ofInstant(Instant.ofEpochMilli(fix.time()), ZoneOffset.UTC)).replace("[UTC]", "");
                String imo = info == null ? "" : info.imo.orElse("");
                System.out.format("%s,%s,%s,%s,\"%s\",%s,%s,%s\n", shapeName, fix.mmsi(), imo, fix.aisClass().name(), type, fix.lat(), fix.lon(), t);
            }).count().toBlocking().single();
        }
    }
}