Examples with Fix au.gov.amsa.risky.format.Fix used on opensource projects

Example 6 with Fix

use of au.gov.amsa.risky.format.Fix in project risky by amsa-code.

the class DistanceTravelledInEezMain method calculateDistance.

private static Observable<? extends Vessel> calculateDistance(File file, Shapefile eezLine, Shapefile eezPolygon, GroupedObservable<Integer, Fix> o) {
    return Observable.defer(() -> {
        State state = new State();
        state.date = file.getName().substring(0, file.getName().indexOf(".track.gz"));
        state.mmsi = o.getKey();
        state.location = Location.UNKNOWN;
        return // 
        o.compose(// 
        Downsample.minTimeStep(5, TimeUnit.MINUTES)).lift(new OperatorEffectiveSpeedChecker(SegmentOptions.builder().acceptAnyFixHours(480L).maxSpeedKnots(50).build())).filter(// 
        check -> check.isOk()).map(// 
        check -> check.fix()).doOnNext(fix -> {
            // TODO unit test
            boolean inside = eezPolygon.contains(fix.lat(), fix.lon());
            Location location = inside ? Location.IN : Location.OUT;
            if (state.location != Location.UNKNOWN) {
                boolean crossed = state.location != location;
                if (crossed) {
                    TimedPosition point = ShapefileUtil.findRegionCrossingPoint(eezLine, state.fix, fix);
                    final double distance;
                    if (location == Location.IN) {
                        distance = distanceKm(fix.lat(), fix.lon(), point.lat, point.lon);
                    } else {
                        distance = distanceKm(state.fix.lat(), state.fix.lon(), point.lat, point.lon);
                    }
                    state.distanceKm += distance;
                    double d = distanceKm(state.fix.lat(), state.fix.lon(), fix.lat(), fix.lon());
                    if (d >= MIN_DISTANCE_KM_TO_ESTIMATE_TIME) {
                        // we ensure that d is not close to zero so that the time estimate does not get
                        // blown out by instability in the division.
                        state.totalTimeMs += distance / d * (fix.time() - state.fix.time());
                    }
                } else if (location == Location.IN) {
                    state.distanceKm += distanceKm(state.fix.lat(), state.fix.lon(), fix.lat(), fix.lon());
                    state.totalTimeMs += fix.time() - state.fix.time();
                }
            }
            state.fix = fix;
            state.location = location;
        }).count().map(count -> new Vessel(count, state));
    });
}

Example 7 with Fix

use of au.gov.amsa.risky.format.Fix in project risky by amsa-code.

the class EffectiveSpeedCheckFailures2Main method main.

public static void main(String[] args) throws ParseException {
    SegmentOptions options = SegmentOptions.builder().acceptAnyFixHours(12L).maxSpeedKnots(50).build();
    tasmania().groupBy(fix -> fix.mmsi()).flatMap(g -> g.lift(new OperatorEffectiveSpeedChecker(options)).buffer(2, 1).filter(list -> list.size() == 2 && list.get(0).isOk() && !list.get(1).isOk()).doOnNext(list -> {
        System.out.println(" ok," + list.get(0));
        System.out.println("bad," + list.get(1));
    })).count().toBlocking().single();
}

Example 8 with Fix

use of au.gov.amsa.risky.format.Fix in project risky by amsa-code.

the class BinaryFixesWithMmsiGzCombinedSortMain method main.

public static void main(String[] args) throws InterruptedException {
    log.info("starting");
    File tracks = new File("/home/dxm/AIS/tracks");
    File combinedSortedTracks = new File("/home/dxm/combinedSortedTracks");
    combinedSortedTracks.mkdir();
    AtomicReference<FileFixes> previous = new AtomicReference<>();
    AtomicInteger n = new AtomicInteger();
    // 
    Observable.from(// 
    tracks.listFiles()).filter(// 
    x -> x.getName().endsWith(".track.gz")).sorted(// 
    (x, y) -> x.getName().compareTo(y.getName())).doOnNext(// 
    x -> log.info("reading " + x)).map(x -> new FileFixes(x, // 
    BinaryFixes.from(x, true, BinaryFixesFormat.WITH_MMSI).toList().toBlocking().first())).doOnNext(ff -> {
        log.info("  extracting previous day fixes from " + ff.file.getName() + ", startTime=" + new Date(ff.startTime));
        FileFixes prev = previous.get();
        if (prev != null) {
            long removed = 0;
            long removedBadTimes = 0;
            long added = 0;
            List<Fix> removeThese = new ArrayList<>(32 * 1024);
            List<Fix> addThese = new ArrayList<>(32 * 1024);
            for (Fix fix : ff.fixes) {
                if (fix.time() < ff.startTime) {
                    removeThese.add(fix);
                    removed++;
                    if (fix.time() >= prev.startTime) {
                        addThese.add(fix);
                        added++;
                    }
                } else if (fix.time() > ff.startTime + TimeUnit.HOURS.toMillis(24)) {
                    // invalid time
                    removeThese.add(fix);
                    removedBadTimes++;
                }
            }
            TreeSet<Fix> set = new TreeSet<Fix>((x, y) -> {
                if (x.time() == y.time()) {
                    return Integer.compare(x.mmsi(), y.mmsi());
                } else {
                    return Long.compare(x.time(), y.time());
                }
            });
            log.info("  building tree set");
            set.addAll(ff.fixes);
            log.info("  removing " + removeThese.size() + " from " + ff.file.getName() + " (badTimes=" + removedBadTimes + ")");
            set.removeAll(removeThese);
            log.info("  copying fixes from set to list");
            ff.fixes = new ArrayList<>(set);
            log.info("  adding " + addThese.size() + " to " + prev.file.getName());
            log.info("  sorting " + prev.file.getName());
            prev.fixes.sort((x, y) -> Long.compare(x.time(), y.time()));
            log.info("  writing gz for " + prev.file.getName());
            writeFixes(combinedSortedTracks, n, prev, removed, added);
        }
        previous.set(ff);
    }).subscribe();
    if (previous.get() != null) {
        writeFixes(combinedSortedTracks, n, previous.get(), 0, 0);
    }
}

Example 9 with Fix

use of au.gov.amsa.risky.format.Fix in project risky by amsa-code.

the class BinaryFixesWithMmsiGzCombinedSortMain method writeFixes.

private static void writeFixes(File combinedSortedTracks, AtomicInteger n, FileFixes prev, long removed, long added) {
    File f = new File(combinedSortedTracks, prev.file.getName());
    try (OutputStream out = new GZIPOutputStream(new FileOutputStream(f))) {
        for (Fix fix : prev.fixes) {
            BinaryFixes.write(fix, out, BinaryFixesFormat.WITH_MMSI);
        }
    } catch (IOException e) {
        throw new UncheckedIOException(e);
    }
    log.info(n.incrementAndGet() + ": removed from next day=" + removed + ", added to this day=" + added + " to " + f);
}

Example 10 with Fix

use of au.gov.amsa.risky.format.Fix in project risky by amsa-code.

the class VoyageDatasetProducer method produce.

public static void produce(File output, File fixesOutput, List<File> list) throws Exception {
    // reset output directories
    output.delete();
    FileUtils.deleteDirectory(fixesOutput);
    int numFiles = list.size();
    System.out.println(numFiles + "binary fix files");
    AtomicInteger fileNumber = new AtomicInteger(0);
    Collection<Port> ports = loadPorts();
    Collection<EezWaypoint> eezWaypoints = readEezWaypoints();
    Shapefile eezLine = Eez.loadEezLine();
    Shapefile eezPolygon = Eez.loadEezPolygon();
    System.out.println("loaded eez shapefiles");
    long t = System.currentTimeMillis();
    AtomicLong failedCheck = new AtomicLong();
    AtomicLong fixCount = new AtomicLong();
    Map<Integer, Integer> mmsisWithFailedChecks = new TreeMap<>();
    Persister persister = new Persister(fixesOutput);
    try (BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(output)))) {
        // Note that in the observable below we don't employ parallel techniques
        // this is because the runtime is acceptable
        // 
        Observable.from(list).groupBy(// 
        f -> mmsiFromFilename(f)).flatMap(files -> {
            String mmsi = files.getKey();
            if (!isShipMmsi(mmsi)) {
                return Observable.empty();
            } else {
                return // 
                files.compose(// 
                o -> logPercentCompleted(numFiles, t, o, fileNumber)).concatMap(// 
                BinaryFixes::from).lift(new OperatorEffectiveSpeedChecker(SegmentOptions.builder().acceptAnyFixHours(24L).maxSpeedKnots(50).build())).doOnNext(// 
                check -> updatedCounts(failedCheck, fixCount, mmsisWithFailedChecks, check)).filter(// 
                check -> check.isOk()).map(// 
                check -> check.fix()).doOnNext(fix -> persister.persist(fix)).compose(// 
                o -> toLegs(eezLine, eezPolygon, ports, eezWaypoints, o)).filter(x -> includeLeg(x));
            }
        }).sorted(// 
        (a, b) -> compareByMmsiThenLegStartTime(a, b)).doOnNext(// 
        x -> write(writer, x)).doOnTerminate(// 
        Checked.a0(() -> persister.close())).toBlocking().subscribe();
        System.out.println((System.currentTimeMillis() - t) + "ms");
        System.out.println("total fixes=" + fixCount.get());
        System.out.println("num fixes rejected due failed effective speed check=" + failedCheck.get());
        System.out.println("num mmsis with failed effective speed checks=" + mmsisWithFailedChecks.size());
        try (PrintStream p = new PrintStream("target/info.txt")) {
            p.println("total fixes=" + fixCount.get());
            p.println("num fixes rejected due failed effective speed check=" + failedCheck.get());
            p.println("num mmsis with failed effective speed checks=" + mmsisWithFailedChecks.size());
        }
        try (PrintStream p = new PrintStream("target/failures.txt")) {
            p.println("failures mmsi <TAB> number of rejected fixes");
            for (Integer mmsi : mmsisWithFailedChecks.keySet()) {
                p.println(mmsi + "\t" + mmsisWithFailedChecks.get(mmsi));
            }
        }
    }
}