Examples with Shapefile au.gov.amsa.gt.Shapefile used on opensource projects

Example 1 with Shapefile

use of au.gov.amsa.gt.Shapefile in project risky by amsa-code.

the class VoyageDatasetProducerTest method test.

@Test
public void test() throws IOException {
    long aTime = 1498199825552L;
    long bTime = aTime + TimeUnit.DAYS.toMillis(2);
    long cTime = bTime + TimeUnit.DAYS.toMillis(4);
    Shapefile eezLine = Eez.loadEezLine();
    Shapefile eezPolygon = Eez.loadEezPolygon();
    assertTrue(eezPolygon.contains(-35, 149));
    assertTrue(!eezPolygon.contains(-35, 175));
    Fix a = createOutOfEez(aTime);
    Fix b = createInEez(bTime);
    PublishSubject<Fix> fixes = PublishSubject.create();
    Collection<Port> ports = VoyageDatasetProducer.loadPorts();
    Collection<EezWaypoint> eezWaypoints = Collections.singleton(new EezWaypoint(EEZ_WAYPOINT_NAME, -35, 151.0, Optional.empty()));
    AssertableSubscriber<TimedLeg> ts = VoyageDatasetProducer.toLegs(eezLine, eezPolygon, ports, eezWaypoints, // 
    fixes).test();
    fixes.onNext(a);
    fixes.onNext(b);
    ts.assertNoValues();
    // in sydney port
    Fix c = createInSydneyPort(cTime);
    fixes.onNext(c);
    // 
    ts.assertNoTerminalEvent().assertValueCount(1);
    {
        TimedLeg leg = ts.getOnNextEvents().get(0);
        System.out.println(leg);
        assertTrue(leg.a.waypoint instanceof EezWaypoint);
        assertTrue(leg.b.waypoint instanceof Port);
        assertEquals(EEZ_WAYPOINT_NAME, leg.a.waypoint.name());
        assertEquals("Sydney", leg.b.waypoint.name());
        assertEquals(cTime, leg.b.time);
        assertTrue(leg.a.time > a.time());
        assertTrue(leg.a.time < b.time());
        assertTrue(leg.a.time < leg.b.time);
    }
    // another sydney report, the next leg should start with this timestamp
    long dTime = cTime + TimeUnit.DAYS.toMillis(1);
    Fix d = createInSydneyPort(dTime);
    fixes.onNext(d);
    // 
    ts.assertNoTerminalEvent().assertValueCount(1);
    // out of sydney
    long eTime = dTime + TimeUnit.DAYS.toMillis(1);
    Fix e = createInEez(eTime);
    fixes.onNext(e);
    // 
    ts.assertNoTerminalEvent().assertValueCount(1);
    // out of eez
    long fTime = eTime + TimeUnit.DAYS.toMillis(1);
    Fix f = createOutOfEez(fTime);
    fixes.onNext(f);
    // 
    ts.assertNoTerminalEvent().assertValueCount(2);
    {
        TimedLeg leg = ts.getOnNextEvents().get(1);
        System.out.println(leg);
        assertTrue(leg.a.waypoint instanceof Port);
        assertEquals("Sydney", leg.a.waypoint.name());
        assertTrue(leg.b.waypoint instanceof EezWaypoint);
        assertEquals(EEZ_WAYPOINT_NAME, leg.b.waypoint.name());
        assertEquals(dTime, leg.a.time);
    }
}

Example 2 with Shapefile

use of au.gov.amsa.gt.Shapefile in project risky by amsa-code.

the class DistanceTravelledInEezMain method main.

public static void main(String[] args) throws FileNotFoundException, IOException {
    System.out.println("running");
    File tracks = new File("/home/dxm/combinedSortedTracks");
    long t = System.currentTimeMillis();
    List<File> files = Arrays.asList(tracks.listFiles());
    files.sort((a, b) -> a.getName().compareTo(b.getName()));
    try (PrintStream out = new PrintStream(new BufferedOutputStream(new FileOutputStream("target/output.csv")))) {
        out.println(Vessel.headings());
        // 
        Observable.from(// 
        files).filter(// 
        x -> x.getName().endsWith(".track.gz")).filter(// 
        x -> x.getName().startsWith("2017")).flatMap(file -> {
            log.info(file);
            // Note that the Shapefile objects are not thread-safe so we make new one for
            // each file to enable parallel processing
            // used for intersections with eez boundary
            Shapefile eezLine = Eez.loadEezLine();
            // used for contains tests
            Shapefile eezPolygon = Eez.loadEezPolygon();
            long startTime = Util.getStartTime(file);
            long endTime = startTime + TimeUnit.HOURS.toMillis(24);
            return // 
            BinaryFixes.from(file, true, // 
            BinaryFixesFormat.WITH_MMSI).subscribeOn(// 
            Schedulers.computation()).filter(// 
            f -> MmsiValidator2.INSTANCE.isValid(f.mmsi())).filter(// 
            f -> f.time() >= startTime && f.time() <= endTime).groupBy(// 
            fix -> fix.mmsi()).flatMap(o -> calculateDistance(file, eezLine, eezPolygon, o));
        }, // 
        Runtime.getRuntime().availableProcessors()).filter(// 
        x -> x.state.fix != null).toBlocking().forEach(x -> out.println(x.line()));
    }
    System.out.println((System.currentTimeMillis() - t) + "ms");
}

Example 3 with Shapefile

use of au.gov.amsa.gt.Shapefile in project risky by amsa-code.

the class DistanceTravelledInEezMain method calculateDistance.

private static Observable<? extends Vessel> calculateDistance(File file, Shapefile eezLine, Shapefile eezPolygon, GroupedObservable<Integer, Fix> o) {
    return Observable.defer(() -> {
        State state = new State();
        state.date = file.getName().substring(0, file.getName().indexOf(".track.gz"));
        state.mmsi = o.getKey();
        state.location = Location.UNKNOWN;
        return // 
        o.compose(// 
        Downsample.minTimeStep(5, TimeUnit.MINUTES)).lift(new OperatorEffectiveSpeedChecker(SegmentOptions.builder().acceptAnyFixHours(480L).maxSpeedKnots(50).build())).filter(// 
        check -> check.isOk()).map(// 
        check -> check.fix()).doOnNext(fix -> {
            // TODO unit test
            boolean inside = eezPolygon.contains(fix.lat(), fix.lon());
            Location location = inside ? Location.IN : Location.OUT;
            if (state.location != Location.UNKNOWN) {
                boolean crossed = state.location != location;
                if (crossed) {
                    TimedPosition point = ShapefileUtil.findRegionCrossingPoint(eezLine, state.fix, fix);
                    final double distance;
                    if (location == Location.IN) {
                        distance = distanceKm(fix.lat(), fix.lon(), point.lat, point.lon);
                    } else {
                        distance = distanceKm(state.fix.lat(), state.fix.lon(), point.lat, point.lon);
                    }
                    state.distanceKm += distance;
                    double d = distanceKm(state.fix.lat(), state.fix.lon(), fix.lat(), fix.lon());
                    if (d >= MIN_DISTANCE_KM_TO_ESTIMATE_TIME) {
                        // we ensure that d is not close to zero so that the time estimate does not get
                        // blown out by instability in the division.
                        state.totalTimeMs += distance / d * (fix.time() - state.fix.time());
                    }
                } else if (location == Location.IN) {
                    state.distanceKm += distanceKm(state.fix.lat(), state.fix.lon(), fix.lat(), fix.lon());
                    state.totalTimeMs += fix.time() - state.fix.time();
                }
            }
            state.fix = fix;
            state.location = location;
        }).count().map(count -> new Vessel(count, state));
    });
}

Example 4 with Shapefile

use of au.gov.amsa.gt.Shapefile in project risky by amsa-code.

the class VoyageDatasetProducer method produce.

public static void produce(File output, File fixesOutput, List<File> list) throws Exception {
    // reset output directories
    output.delete();
    FileUtils.deleteDirectory(fixesOutput);
    int numFiles = list.size();
    System.out.println(numFiles + "binary fix files");
    AtomicInteger fileNumber = new AtomicInteger(0);
    Collection<Port> ports = loadPorts();
    Collection<EezWaypoint> eezWaypoints = readEezWaypoints();
    Shapefile eezLine = Eez.loadEezLine();
    Shapefile eezPolygon = Eez.loadEezPolygon();
    System.out.println("loaded eez shapefiles");
    long t = System.currentTimeMillis();
    AtomicLong failedCheck = new AtomicLong();
    AtomicLong fixCount = new AtomicLong();
    Map<Integer, Integer> mmsisWithFailedChecks = new TreeMap<>();
    Persister persister = new Persister(fixesOutput);
    try (BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(output)))) {
        // Note that in the observable below we don't employ parallel techniques
        // this is because the runtime is acceptable
        // 
        Observable.from(list).groupBy(// 
        f -> mmsiFromFilename(f)).flatMap(files -> {
            String mmsi = files.getKey();
            if (!isShipMmsi(mmsi)) {
                return Observable.empty();
            } else {
                return // 
                files.compose(// 
                o -> logPercentCompleted(numFiles, t, o, fileNumber)).concatMap(// 
                BinaryFixes::from).lift(new OperatorEffectiveSpeedChecker(SegmentOptions.builder().acceptAnyFixHours(24L).maxSpeedKnots(50).build())).doOnNext(// 
                check -> updatedCounts(failedCheck, fixCount, mmsisWithFailedChecks, check)).filter(// 
                check -> check.isOk()).map(// 
                check -> check.fix()).doOnNext(fix -> persister.persist(fix)).compose(// 
                o -> toLegs(eezLine, eezPolygon, ports, eezWaypoints, o)).filter(x -> includeLeg(x));
            }
        }).sorted(// 
        (a, b) -> compareByMmsiThenLegStartTime(a, b)).doOnNext(// 
        x -> write(writer, x)).doOnTerminate(// 
        Checked.a0(() -> persister.close())).toBlocking().subscribe();
        System.out.println((System.currentTimeMillis() - t) + "ms");
        System.out.println("total fixes=" + fixCount.get());
        System.out.println("num fixes rejected due failed effective speed check=" + failedCheck.get());
        System.out.println("num mmsis with failed effective speed checks=" + mmsisWithFailedChecks.size());
        try (PrintStream p = new PrintStream("target/info.txt")) {
            p.println("total fixes=" + fixCount.get());
            p.println("num fixes rejected due failed effective speed check=" + failedCheck.get());
            p.println("num mmsis with failed effective speed checks=" + mmsisWithFailedChecks.size());
        }
        try (PrintStream p = new PrintStream("target/failures.txt")) {
            p.println("failures mmsi <TAB> number of rejected fixes");
            for (Integer mmsi : mmsisWithFailedChecks.keySet()) {
                p.println(mmsi + "\t" + mmsisWithFailedChecks.get(mmsi));
            }
        }
    }
}

Example 5 with Shapefile

use of au.gov.amsa.gt.Shapefile in project risky by amsa-code.

the class CountCrossingsIntoRegionMain method main.

public static void main(String[] args) {
    String shape = "/home/dxm/temp/amsa_atba_ningaloo_reef_pl.shp";
    // String shape = "/home/dxm/temp/amb06_map_eez_pl.shp";
    Shapefile region = Shapefile.from(new File(shape));
    // test load
    region.contains(0, 0);
    Func2<Fix, Fix, Integer> ascendingTime = (a, b) -> ((Long) a.time()).compareTo(b.time());
    Map<Integer, Info> ships = ShipStaticData.getMapFromResource("/ship-data.txt");
    Pattern pattern = Pattern.compile(".*\\.track");
    List<File> files = Files.find(new File("/media/an/binary-fixes-5-minute/2012"), pattern);
    files.addAll(Files.find(new File("/media/an/binary-fixes-5-minute/2013"), pattern));
    files.addAll(Files.find(new File("/media/an/binary-fixes-5-minute/2014"), pattern));
    files.addAll(Files.find(new File("/media/an/binary-fixes-5-minute/2015"), pattern));
    log.info("files=" + files.size());
    int count = Observable.from(files).concatMap(file -> detectCrossings(file, region)).toSortedList(ascendingTime).flatMap(o -> Observable.from(o)).doOnNext(fix -> {
        Info info = ships.get(fix.mmsi());
        if (info != null) {
            String type;
            if (info.shipType.isPresent())
                type = ShipTypeDecoder.getShipType(info.shipType.get());
            else
                type = "UNKNOWN";
            String t = DateTimeFormatter.ISO_DATE_TIME.format(ZonedDateTime.ofInstant(Instant.ofEpochMilli(fix.time()), ZoneId.of("UTC")));
            System.out.println(t + ", mmsi=" + fix.mmsi() + ", class=" + fix.aisClass().name() + ", lengthMetres=" + info.lengthMetres().orElse(null) + ", type=" + type + ", lat=" + fix.lat() + ", lon=" + fix.lon());
        }
    }).count().toBlocking().single();
    System.out.println("count=" + count);
}