Search in sources :

Example 1 with OperatorEffectiveSpeedChecker

use of au.gov.amsa.geo.distance.OperatorEffectiveSpeedChecker in project risky by amsa-code.

the class DistanceTravelledInEezMain method calculateDistance.

private static Observable<? extends Vessel> calculateDistance(File file, Shapefile eezLine, Shapefile eezPolygon, GroupedObservable<Integer, Fix> o) {
    return Observable.defer(() -> {
        State state = new State();
        state.date = file.getName().substring(0, file.getName().indexOf(".track.gz"));
        state.mmsi = o.getKey();
        state.location = Location.UNKNOWN;
        return // 
        o.compose(// 
        Downsample.minTimeStep(5, TimeUnit.MINUTES)).lift(new OperatorEffectiveSpeedChecker(SegmentOptions.builder().acceptAnyFixHours(480L).maxSpeedKnots(50).build())).filter(// 
        check -> check.isOk()).map(// 
        check -> check.fix()).doOnNext(fix -> {
            // TODO unit test
            boolean inside = eezPolygon.contains(fix.lat(), fix.lon());
            Location location = inside ? Location.IN : Location.OUT;
            if (state.location != Location.UNKNOWN) {
                boolean crossed = state.location != location;
                if (crossed) {
                    TimedPosition point = ShapefileUtil.findRegionCrossingPoint(eezLine, state.fix, fix);
                    final double distance;
                    if (location == Location.IN) {
                        distance = distanceKm(fix.lat(), fix.lon(), point.lat, point.lon);
                    } else {
                        distance = distanceKm(state.fix.lat(), state.fix.lon(), point.lat, point.lon);
                    }
                    state.distanceKm += distance;
                    double d = distanceKm(state.fix.lat(), state.fix.lon(), fix.lat(), fix.lon());
                    if (d >= MIN_DISTANCE_KM_TO_ESTIMATE_TIME) {
                        // we ensure that d is not close to zero so that the time estimate does not get
                        // blown out by instability in the division.
                        state.totalTimeMs += distance / d * (fix.time() - state.fix.time());
                    }
                } else if (location == Location.IN) {
                    state.distanceKm += distanceKm(state.fix.lat(), state.fix.lon(), fix.lat(), fix.lon());
                    state.totalTimeMs += fix.time() - state.fix.time();
                }
            }
            state.fix = fix;
            state.location = location;
        }).count().map(count -> new Vessel(count, state));
    });
}
Also used : Arrays(java.util.Arrays) Downsample(au.gov.amsa.risky.format.Downsample) Date(java.util.Date) SegmentOptions(au.gov.amsa.geo.model.SegmentOptions) OperatorEffectiveSpeedChecker(au.gov.amsa.geo.distance.OperatorEffectiveSpeedChecker) SimpleDateFormat(java.text.SimpleDateFormat) ShapefileUtil(au.gov.amsa.geo.ShapefileUtil) BufferedOutputStream(java.io.BufferedOutputStream) TimedPosition(au.gov.amsa.geo.TimedPosition) Observable(rx.Observable) Logger(org.apache.log4j.Logger) BinaryFixes(au.gov.amsa.risky.format.BinaryFixes) Fix(au.gov.amsa.risky.format.Fix) Schedulers(rx.schedulers.Schedulers) Eez(au.gov.amsa.geo.Eez) ParseException(java.text.ParseException) PrintStream(java.io.PrintStream) TimeZone(java.util.TimeZone) DecimalFormat(java.text.DecimalFormat) FileOutputStream(java.io.FileOutputStream) IOException(java.io.IOException) File(java.io.File) FileNotFoundException(java.io.FileNotFoundException) BinaryFixesFormat(au.gov.amsa.risky.format.BinaryFixesFormat) TimeUnit(java.util.concurrent.TimeUnit) Position(com.github.davidmoten.grumpy.core.Position) List(java.util.List) GroupedObservable(rx.observables.GroupedObservable) Shapefile(au.gov.amsa.gt.Shapefile) MmsiValidator2(au.gov.amsa.util.identity.MmsiValidator2) OperatorEffectiveSpeedChecker(au.gov.amsa.geo.distance.OperatorEffectiveSpeedChecker) TimedPosition(au.gov.amsa.geo.TimedPosition)

Example 2 with OperatorEffectiveSpeedChecker

use of au.gov.amsa.geo.distance.OperatorEffectiveSpeedChecker in project risky by amsa-code.

the class EffectiveSpeedCheckFailures2Main method main.

public static void main(String[] args) throws ParseException {
    SegmentOptions options = SegmentOptions.builder().acceptAnyFixHours(12L).maxSpeedKnots(50).build();
    tasmania().groupBy(fix -> fix.mmsi()).flatMap(g -> g.lift(new OperatorEffectiveSpeedChecker(options)).buffer(2, 1).filter(list -> list.size() == 2 && list.get(0).isOk() && !list.get(1).isOk()).doOnNext(list -> {
        System.out.println(" ok," + list.get(0));
        System.out.println("bad," + list.get(1));
    })).count().toBlocking().single();
}
Also used : Arrays(java.util.Arrays) BinaryFixes(au.gov.amsa.risky.format.BinaryFixes) Func2(rx.functions.Func2) SegmentOptions(au.gov.amsa.geo.model.SegmentOptions) OperatorEffectiveSpeedChecker(au.gov.amsa.geo.distance.OperatorEffectiveSpeedChecker) Fix(au.gov.amsa.risky.format.Fix) ParseException(java.text.ParseException) File(java.io.File) Observable(rx.Observable) SegmentOptions(au.gov.amsa.geo.model.SegmentOptions) OperatorEffectiveSpeedChecker(au.gov.amsa.geo.distance.OperatorEffectiveSpeedChecker)

Example 3 with OperatorEffectiveSpeedChecker

use of au.gov.amsa.geo.distance.OperatorEffectiveSpeedChecker in project risky by amsa-code.

the class EffectiveSpeedFailuresMain method main.

public static void main(String[] args) {
    Pattern pattern = Pattern.compile(".*\\.track");
    List<File> files = Files.find(new File("/media/an/binary-fixes-5-minute/2015"), pattern);
    int count = Observable.from(files).filter(file -> !file.getName().equals("0.track")).flatMap(file -> BinaryFixes.from(file).lift(new OperatorEffectiveSpeedChecker(SegmentOptions.builder().acceptAnyFixHours(12L).maxSpeedKnots(50).build())).filter(check -> !check.isOk()).reduce(new MmsiCount(0, 0), (mc, fix) -> new MmsiCount(fix.fix().mmsi(), mc.count + 1)).filter(mc -> mc.count >= 1000)).toSortedList((a, b) -> Long.compare(b.count, a.count)).flatMapIterable(x -> x).doOnNext(mc -> System.out.println(mc.mmsi)).count().toBlocking().single();
    System.out.println(count);
}
Also used : List(java.util.List) BinaryFixes(au.gov.amsa.risky.format.BinaryFixes) SegmentOptions(au.gov.amsa.geo.model.SegmentOptions) OperatorEffectiveSpeedChecker(au.gov.amsa.geo.distance.OperatorEffectiveSpeedChecker) Pattern(java.util.regex.Pattern) Files(au.gov.amsa.util.Files) File(java.io.File) Observable(rx.Observable) Pattern(java.util.regex.Pattern) OperatorEffectiveSpeedChecker(au.gov.amsa.geo.distance.OperatorEffectiveSpeedChecker) File(java.io.File)

Example 4 with OperatorEffectiveSpeedChecker

use of au.gov.amsa.geo.distance.OperatorEffectiveSpeedChecker in project risky by amsa-code.

the class VoyageDatasetProducer method produce.

public static void produce(File output, File fixesOutput, List<File> list) throws Exception {
    // reset output directories
    output.delete();
    FileUtils.deleteDirectory(fixesOutput);
    int numFiles = list.size();
    System.out.println(numFiles + "binary fix files");
    AtomicInteger fileNumber = new AtomicInteger(0);
    Collection<Port> ports = loadPorts();
    Collection<EezWaypoint> eezWaypoints = readEezWaypoints();
    Shapefile eezLine = Eez.loadEezLine();
    Shapefile eezPolygon = Eez.loadEezPolygon();
    System.out.println("loaded eez shapefiles");
    long t = System.currentTimeMillis();
    AtomicLong failedCheck = new AtomicLong();
    AtomicLong fixCount = new AtomicLong();
    Map<Integer, Integer> mmsisWithFailedChecks = new TreeMap<>();
    Persister persister = new Persister(fixesOutput);
    try (BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(output)))) {
        // Note that in the observable below we don't employ parallel techniques
        // this is because the runtime is acceptable
        // 
        Observable.from(list).groupBy(// 
        f -> mmsiFromFilename(f)).flatMap(files -> {
            String mmsi = files.getKey();
            if (!isShipMmsi(mmsi)) {
                return Observable.empty();
            } else {
                return // 
                files.compose(// 
                o -> logPercentCompleted(numFiles, t, o, fileNumber)).concatMap(// 
                BinaryFixes::from).lift(new OperatorEffectiveSpeedChecker(SegmentOptions.builder().acceptAnyFixHours(24L).maxSpeedKnots(50).build())).doOnNext(// 
                check -> updatedCounts(failedCheck, fixCount, mmsisWithFailedChecks, check)).filter(// 
                check -> check.isOk()).map(// 
                check -> check.fix()).doOnNext(fix -> persister.persist(fix)).compose(// 
                o -> toLegs(eezLine, eezPolygon, ports, eezWaypoints, o)).filter(x -> includeLeg(x));
            }
        }).sorted(// 
        (a, b) -> compareByMmsiThenLegStartTime(a, b)).doOnNext(// 
        x -> write(writer, x)).doOnTerminate(// 
        Checked.a0(() -> persister.close())).toBlocking().subscribe();
        System.out.println((System.currentTimeMillis() - t) + "ms");
        System.out.println("total fixes=" + fixCount.get());
        System.out.println("num fixes rejected due failed effective speed check=" + failedCheck.get());
        System.out.println("num mmsis with failed effective speed checks=" + mmsisWithFailedChecks.size());
        try (PrintStream p = new PrintStream("target/info.txt")) {
            p.println("total fixes=" + fixCount.get());
            p.println("num fixes rejected due failed effective speed check=" + failedCheck.get());
            p.println("num mmsis with failed effective speed checks=" + mmsisWithFailedChecks.size());
        }
        try (PrintStream p = new PrintStream("target/failures.txt")) {
            p.println("failures mmsi <TAB> number of rejected fixes");
            for (Integer mmsi : mmsisWithFailedChecks.keySet()) {
                p.println(mmsi + "\t" + mmsisWithFailedChecks.get(mmsi));
            }
        }
    }
}
Also used : SegmentOptions(au.gov.amsa.geo.model.SegmentOptions) ZonedDateTime(java.time.ZonedDateTime) OperatorEffectiveSpeedChecker(au.gov.amsa.geo.distance.OperatorEffectiveSpeedChecker) Preconditions(com.github.davidmoten.guavamini.Preconditions) BufferedOutputStream(java.io.BufferedOutputStream) ArrayList(java.util.ArrayList) Observable(rx.Observable) BinaryFixes(au.gov.amsa.risky.format.BinaryFixes) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Map(java.util.Map) Fix(au.gov.amsa.risky.format.Fix) OutputStreamWriter(java.io.OutputStreamWriter) ZoneOffset(java.time.ZoneOffset) OutputStream(java.io.OutputStream) PrintStream(java.io.PrintStream) BufferedWriter(java.io.BufferedWriter) Collection(java.util.Collection) DecimalFormat(java.text.DecimalFormat) FileOutputStream(java.io.FileOutputStream) IOException(java.io.IOException) FileUtils(org.apache.commons.io.FileUtils) EffectiveSpeedCheck(au.gov.amsa.geo.distance.EffectiveSpeedCheck) Reader(java.io.Reader) Checked(com.github.davidmoten.rx.Checked) Instant(java.time.Instant) InputStreamReader(java.io.InputStreamReader) Files(au.gov.amsa.util.Files) File(java.io.File) FileNotFoundException(java.io.FileNotFoundException) BinaryFixesFormat(au.gov.amsa.risky.format.BinaryFixesFormat) Strings(au.gov.amsa.streams.Strings) Position(com.github.davidmoten.grumpy.core.Position) VisibleForTesting(com.github.davidmoten.guavamini.annotations.VisibleForTesting) AtomicLong(java.util.concurrent.atomic.AtomicLong) List(java.util.List) TreeMap(java.util.TreeMap) Closeable(java.io.Closeable) DateTimeFormatter(java.time.format.DateTimeFormatter) Optional(java.util.Optional) Shapefile(au.gov.amsa.gt.Shapefile) Pattern(java.util.regex.Pattern) PrintStream(java.io.PrintStream) OperatorEffectiveSpeedChecker(au.gov.amsa.geo.distance.OperatorEffectiveSpeedChecker) TreeMap(java.util.TreeMap) BufferedWriter(java.io.BufferedWriter) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) AtomicLong(java.util.concurrent.atomic.AtomicLong) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) FileOutputStream(java.io.FileOutputStream) Shapefile(au.gov.amsa.gt.Shapefile) BinaryFixes(au.gov.amsa.risky.format.BinaryFixes) OutputStreamWriter(java.io.OutputStreamWriter)

Aggregations

OperatorEffectiveSpeedChecker (au.gov.amsa.geo.distance.OperatorEffectiveSpeedChecker)4 SegmentOptions (au.gov.amsa.geo.model.SegmentOptions)4 BinaryFixes (au.gov.amsa.risky.format.BinaryFixes)4 File (java.io.File)4 Observable (rx.Observable)4 Fix (au.gov.amsa.risky.format.Fix)3 List (java.util.List)3 Shapefile (au.gov.amsa.gt.Shapefile)2 BinaryFixesFormat (au.gov.amsa.risky.format.BinaryFixesFormat)2 Files (au.gov.amsa.util.Files)2 Position (com.github.davidmoten.grumpy.core.Position)2 BufferedOutputStream (java.io.BufferedOutputStream)2 FileNotFoundException (java.io.FileNotFoundException)2 FileOutputStream (java.io.FileOutputStream)2 IOException (java.io.IOException)2 PrintStream (java.io.PrintStream)2 DecimalFormat (java.text.DecimalFormat)2 ParseException (java.text.ParseException)2 Arrays (java.util.Arrays)2 Pattern (java.util.regex.Pattern)2