Examples with Files au.gov.amsa.util.Files used on opensource projects

Example 6 with Files

use of au.gov.amsa.util.Files in project risky by amsa-code.

the class CountCrossingsIntoRegionMain method main.

public static void main(String[] args) {
    String shape = "/home/dxm/temp/amsa_atba_ningaloo_reef_pl.shp";
    // String shape = "/home/dxm/temp/amb06_map_eez_pl.shp";
    Shapefile region = Shapefile.from(new File(shape));
    // test load
    region.contains(0, 0);
    Func2<Fix, Fix, Integer> ascendingTime = (a, b) -> ((Long) a.time()).compareTo(b.time());
    Map<Integer, Info> ships = ShipStaticData.getMapFromResource("/ship-data.txt");
    Pattern pattern = Pattern.compile(".*\\.track");
    List<File> files = Files.find(new File("/media/an/binary-fixes-5-minute/2012"), pattern);
    files.addAll(Files.find(new File("/media/an/binary-fixes-5-minute/2013"), pattern));
    files.addAll(Files.find(new File("/media/an/binary-fixes-5-minute/2014"), pattern));
    files.addAll(Files.find(new File("/media/an/binary-fixes-5-minute/2015"), pattern));
    log.info("files=" + files.size());
    int count = Observable.from(files).concatMap(file -> detectCrossings(file, region)).toSortedList(ascendingTime).flatMap(o -> Observable.from(o)).doOnNext(fix -> {
        Info info = ships.get(fix.mmsi());
        if (info != null) {
            String type;
            if (info.shipType.isPresent())
                type = ShipTypeDecoder.getShipType(info.shipType.get());
            else
                type = "UNKNOWN";
            String t = DateTimeFormatter.ISO_DATE_TIME.format(ZonedDateTime.ofInstant(Instant.ofEpochMilli(fix.time()), ZoneId.of("UTC")));
            System.out.println(t + ", mmsi=" + fix.mmsi() + ", class=" + fix.aisClass().name() + ", lengthMetres=" + info.lengthMetres().orElse(null) + ", type=" + type + ", lat=" + fix.lat() + ", lon=" + fix.lon());
        }
    }).count().toBlocking().single();
    System.out.println("count=" + count);
}

Example 7 with Files

use of au.gov.amsa.util.Files in project risky by amsa-code.

the class GeoffHAdhocMain method main.

public static void main(String[] args) {
    int option = 1;
    if (option == 1) {
        String filename = "/home/dxm/temp/235099876.track";
        // String filename =
        // "/media/an/binary-fixes-5-minute/2014/235099876.track";
        File file = new File(filename);
        BinaryFixes.from(file).doOnNext(System.out::println).toSortedList((a, b) -> Long.compare(a.time(), b.time())).flatMap(Observable::from).doOnCompleted(() -> System.out.println("complete")).subscribe();
    } else if (option == 2) {
        Observable<File> files = Observable.from(Files.find(new File("/media/an/binary-fixes-5-minute/2014"), Pattern.compile("\\d+\\.track")));
        Func2<Pair<File, Long>, Pair<File, Long>, Pair<File, Long>> earliest = (p1, p2) -> {
            if (p1.b() <= p2.b())
                return p1;
            else
                return p2;
        };
        files.filter(file -> !file.getName().equals("0.track")).flatMap(file -> BinaryFixes.from(file).<Pair<File, Long>>map(fix -> Pair.create(file, fix.time())).reduce(earliest).doOnNext(p -> System.out.println(file + ": " + p)).subscribeOn(Schedulers.computation())).reduce(earliest).doOnNext(System.out::println).toBlocking().single();
    }
}

Example 8 with Files

use of au.gov.amsa.util.Files in project risky by amsa-code.

the class VesselsInCoverageMain method main.

public static void main(String[] args) throws IOException {
    long t = System.currentTimeMillis();
    File out = new File("target/mmsi.txt");
    out.delete();
    try (BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(out)))) {
        Pattern pattern = Pattern.compile(".*\\.track");
        List<File> list = new ArrayList<File>();
        list.addAll(Files.find(new File("/media/an/binary-fixes-5-minute/2014"), pattern));
        list.addAll(Files.find(new File("/media/an/binary-fixes-5-minute/2015"), pattern));
        list.addAll(Files.find(new File("/media/an/binary-fixes-5-minute/2016"), pattern));
        AtomicInteger count = new AtomicInteger();
        // 
        Observable.from(list).groupBy(f -> count.getAndIncrement() % // 
        Runtime.getRuntime().availableProcessors()).flatMap(// 
        files -> toFixes(files, Schedulers.computation())).distinct(// 
        fix -> fix.mmsi() + fix.aisClass().name()).sorted(// 
        (a, b) -> Integer.compare(a.mmsi(), b.mmsi())).filter(// 
        fix -> MmsiValidator2.INSTANCE.isValid((long) fix.mmsi())).doOnNext(// 
        fix -> write(writer, fix)).toBlocking().subscribe();
    }
    System.out.println((System.currentTimeMillis() - t) + "ms");
}

Example 9 with Files

use of au.gov.amsa.util.Files in project risky by amsa-code.

the class HeadingCourseDiffMain method main.

public static void main(String[] args) {
    Pattern pattern = Pattern.compile(".*\\.track");
    List<File> files = Files.find(new File("/media/an/binary-fixes-5-minute/2014"), pattern);
    // files.addAll(Files.find(new
    // File("/media/an/binary-fixes-5-minute/2013"), pattern));
    // files.addAll(Files.find(new
    // File("/media/an/binary-fixes-5-minute/2014"), pattern));
    // files.addAll(Files.find(new
    // File("/media/an/binary-fixes-5-minute/2015"), pattern));
    log.info("files=" + files.size());
    ConcurrentHashMap<Integer, Long> map = new ConcurrentHashMap<>();
    int count = // 
    Observable.from(files).concatMap(file -> BinaryFixes.from(file).filter(f -> f.courseOverGroundDegrees().isPresent() && f.headingDegrees().isPresent()).map(f -> Math.abs(Position.getBearingDifferenceDegrees(f.courseOverGroundDegrees().get(), // 
    f.headingDegrees().get()))).doOnNext(x -> {
        int bucket = (int) Math.round(x);
        map.compute(bucket, (n, num) -> num == null ? 1 : num + 1);
    })).count().toBlocking().single();
    System.out.println("count=" + count);
    for (Integer key : new TreeSet<Integer>(map.keySet())) {
        System.out.println(key + "\t" + map.get(key));
    }
}

Example 10 with Files

use of au.gov.amsa.util.Files in project risky by amsa-code.

the class ByMmsiToDailyConverter method convert.

public static void convert(File input, File output) {
    DateTimeFormatter dtf = DateTimeFormatter.ofPattern("yyyy-MM-dd").withZone(ZoneOffset.UTC);
    output.mkdirs();
    try {
        FileUtils.cleanDirectory(output);
    } catch (IOException e1) {
        throw new RuntimeException(e1);
    }
    {
        List<File> files = Files.find(input, Pattern.compile(".*\\.track"));
        System.out.println("found " + files.size() + " files");
        int bufferSize = 1000;
        Observable.from(files).flatMap(file -> BinaryFixes.from(file, true)).lift(Logging.<Fix>logger().showMemory().every(1000000).showCount("recordsMillions").log()).groupBy(fix -> dtf.format(Instant.ofEpochMilli(fix.time()))).flatMap(g -> g.buffer(bufferSize)).doOnNext(list -> {
            File file = new File(output, dtf.format(Instant.ofEpochMilli(list.get(0).time())) + ".fix");
            try (OutputStream os = new FileOutputStream(file, true)) {
                for (Fix fix : list) {
                    BinaryFixes.write(fix, os, BinaryFixesFormat.WITH_MMSI);
                }
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
        }).count().toBlocking().single();
    }
}