Examples with Logging com.github.davidmoten.rx.slf4j.Logging used on opensource projects

Example 1 with Logging

use of com.github.davidmoten.rx.slf4j.Logging in project risky by amsa-code.

the class BinaryFixesMain method main.

public static void main(String[] args) {
    // perform a speed test for loading BinaryFixes from disk
    FixImpl.validate = false;
    final ConcurrentHashMap<Long, List<FixImpl>> map = new ConcurrentHashMap<Long, List<FixImpl>>();
    // -downsample-5-mins
    List<File> files = Files.find(new File("/media/an/binary-fixes/2014-year-downsample-5-mins"), Pattern.compile(".*\\.track"));
    long t = System.currentTimeMillis();
    long count = Observable.from(files).buffer(Math.max(1, files.size() / Runtime.getRuntime().availableProcessors())).flatMap(list -> {
        return Observable.from(list).concatMap(file -> BinaryFixes.from(file).countLong()).subscribeOn(Schedulers.computation());
    }).scan(0L, (a, b) -> a + b).lift(Logging.<Long>logger().showCount().prefix("records=").showMemory().every(1000).log()).last().toBlocking().single();
    long elapsed = System.currentTimeMillis() - t;
    System.out.println("Map size = " + map.size());
    System.out.println("Total records = " + count + ", numPerSecond=" + count * 1000.0 / elapsed + ", timeMs=" + elapsed);
}

Example 2 with Logging

use of com.github.davidmoten.rx.slf4j.Logging in project risky by amsa-code.

the class ShipTypeBreakdownMain method main.

public static void main(String[] args) throws FileNotFoundException, IOException {
    // load a shapefile
    final GeometryFactory gf = new GeometryFactory();
    File file = new File("/home/dxm/temp/srr.shp");
    Map<String, Serializable> map = new HashMap<>();
    map.put("url", file.toURI().toURL());
    DataStore datastore = DataStoreFinder.getDataStore(map);
    String typeName = datastore.getTypeNames()[0];
    System.out.println(typeName);
    SimpleFeatureSource source = datastore.getFeatureSource(typeName);
    final SimpleFeatureCollection features = source.getFeatures();
    final List<PreparedGeometry> geometries = new ArrayList<>();
    SimpleFeatureIterator it = features.features();
    while (it.hasNext()) {
        SimpleFeature feature = it.next();
        Geometry g = (Geometry) feature.getDefaultGeometry();
        geometries.add(PreparedGeometryFactory.prepare(g));
    }
    // System.exit(0);
    String filename = "/media/analysis/nmea/2014/NMEA_ITU_20140815.gz";
    final Set<Long> mmsi = new HashSet<Long>();
    final Set<Long> mmsiA = new HashSet<Long>();
    final Set<Long> mmsiB = new HashSet<Long>();
    Streams.extract(Streams.nmeaFromGzip(filename)).flatMap(aisPositionsOnly).lift(Logging.<TimestampedAndLine<AisPosition>>logger().showCount().every(100000).log()).doOnNext(m -> {
        AisPosition p = m.getMessage().get().message();
        if (p.getLatitude() != null && p.getLongitude() != null && contains(gf, geometries, p.getLatitude(), p.getLongitude())) {
            long mmsiNo = m.getMessage().get().message().getMmsi();
            mmsi.add(mmsiNo);
            if (m.getMessage().get().message() instanceof AisPositionA)
                mmsiA.add(mmsiNo);
            else
                mmsiB.add(mmsiNo);
        }
    }).subscribe();
    final Map<ShipTypeClass, Set<Integer>> countsByShipType = new ConcurrentHashMap<>();
    Streams.extract(Streams.nmeaFromGzip(filename)).flatMap(aisShipStaticOnly).doOnNext(m -> {
        AisShipStatic s = m.getMessage().get().message();
        if (mmsi.contains(Long.valueOf(s.getMmsi()))) {
            boolean isClassA = s instanceof AisShipStaticA;
            ShipTypeClass shipTypeClass = new ShipTypeClass(isClassA, s.getShipType());
            if (countsByShipType.get(shipTypeClass) == null)
                countsByShipType.put(shipTypeClass, new HashSet<Integer>());
            else
                countsByShipType.get(shipTypeClass).add(s.getMmsi());
        }
    }).subscribe();
    System.out.println(countsByShipType);
    Set<String> set = new TreeSet<String>();
    int sizeA = 0;
    int sizeB = 0;
    for (Entry<ShipTypeClass, Set<Integer>> s : countsByShipType.entrySet()) {
        set.add(ShipTypeDecoder.getShipType(s.getKey().shipType) + "\t" + (s.getKey().isClassA ? "A" : "B") + "\t" + s.getValue().size());
        if (s.getKey().isClassA)
            sizeA += s.getValue().size();
        else
            sizeB += s.getValue().size();
    }
    set.stream().forEach(System.out::println);
    System.out.println("Unknown\tA\t" + (mmsiA.size() - sizeA));
    System.out.println("Unknown\tB\t" + (mmsiB.size() - sizeB));
    log.info("finished");
}

Example 3 with Logging

use of com.github.davidmoten.rx.slf4j.Logging in project risky by amsa-code.

the class Collator method main.

public static void main(String[] args) throws IOException {
    final PrintWriter out = new PrintWriter(new OutputStreamWriter(new FileOutputStream("target/destinations.txt"), Charset.forName("UTF-8")));
    File[] files = new File("/media/analysis/nmea/2013").listFiles(f -> f.getName().endsWith(".gz"));
    Arrays.sort(files, (f1, f2) -> f1.getName().compareTo(f2.getName()));
    Observable<File> fileList = Observable.from(files).lift(Logging.<File>logger().showValue().log());
    final AtomicInteger count = new AtomicInteger();
    Observable<Observable<String>> nmeas = Streams.nmeasFromGzip(fileList);
    nmeas.flatMap(nmea -> getDestinations(nmea.doOnNext(line -> {
        int n = count.incrementAndGet();
        if (n % 1000000 == 0)
            System.out.println("lines read=" + (n / 1000000) + "m");
    }))).filter(line -> {
        if (line == null)
            System.out.println("line is null!");
        return line != null;
    }).distinct().doOnNext(destination -> {
        out.println(destination);
        out.flush();
    }).lift(Logging.<String>logger().showCount().showValue().showMemory().every(100).log()).count().toBlocking().single();
    out.close();
}

Example 4 with Logging

use of com.github.davidmoten.rx.slf4j.Logging in project risky by amsa-code.

the class ShipStaticDataCreator method writeStaticDataToFileWithTimestamps.

public static Observable<Timestamped<AisShipStatic>> writeStaticDataToFileWithTimestamps(List<File> files, File outputFile, Scheduler scheduler) {
    SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
    sdf.setTimeZone(TimeZone.getTimeZone("UTC"));
    Func0<PrintStream> resourceFactory = Checked.f0(() -> new PrintStream(outputFile));
    Func1<PrintStream, Observable<Timestamped<AisShipStatic>>> observableFactory = out -> Observable.from(files).buffer(Math.max(1, files.size() / Runtime.getRuntime().availableProcessors() - 1)).flatMap(list -> // 
    Observable.from(list).lift(// 
    Logging.<File>logger().showValue().showMemory().log()).concatMap(// 
    file -> timestampedShipStatics(file))).groupBy(// 
    m -> m.message().getMmsi()).flatMap(// 
    g -> collect(g).subscribeOn(scheduler)).compose(Transformers.doOnFirst(x -> {
        out.println("# MMSI, Time, IMO, AisClass, AisShipType, MaxPresentStaticDraughtMetres, DimAMetres, DimBMetres, DimCMetres, DimDMetres, LengthMetres, WidthMetres, Name");
        out.println("# columns are tab delimited");
        out.println("# -1 = not present");
    })).filter(// 
    set -> set.size() <= 10).flatMapIterable(// 
    set -> set).doOnNext(k -> {
        AisShipStatic m = k.message();
        out.format("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", m.getMmsi(), sdf.format(new Date(k.time())), getImo(m).orElse(-1), m instanceof AisShipStaticA ? "A" : "B", m.getShipType(), getMaximumPresentStaticDraughtMetres(m).orElse(-1F), m.getDimensionA().orElse(-1), m.getDimensionB().orElse(-1), m.getDimensionC().orElse(-1), m.getDimensionD().orElse(-1), AisShipStaticUtil.lengthMetres(m).orElse(-1), AisShipStaticUtil.widthMetres(m).orElse(-1), prepareName(m.getName()));
        out.flush();
    });
    Action1<PrintStream> disposeAction = out -> out.close();
    return Observable.using(resourceFactory, observableFactory, disposeAction);
}

Example 5 with Logging

use of com.github.davidmoten.rx.slf4j.Logging in project risky by amsa-code.

the class CollisionDetectorMain method main.

public static void main(String[] args) throws IOException, InterruptedException {
    VesselPosition.validate = true;
    CollisionDetector c = new CollisionDetector();
    // String filename = "/media/an/nmea/2013/NMEA_ITU_20130108.gz";
    Map<Integer, Info> ships = ShipStaticData.getMapFromResource("/ship-data-2014.txt");
    // nmea from file
    // Streams.nmeaFromGzip(filename)
    File file = new File("/media/an/daily-fixes/2014/2014-02-01.fix");
    File candidates = new File("/media/an/temp/" + file.getName() + ".collision-candidates.txt");
    try (PrintStream out = new PrintStream(candidates)) {
        out.println("time,mmsi1,lat1, lon1, cog1, m/s, mmsi2, lat2, lon2, cog2, m/s");
        BinaryFixes.from(file, true, BinaryFixesFormat.WITH_MMSI).map(VesselPositions.TO_VESSEL_POSITION).lift(Logging.<VesselPosition>logger().showCount().every(1000).showMemory().log()).filter(onlyClassA).filter(p -> p.speedMetresPerSecond().isPresent()).filter(p -> p.cogDegrees().isPresent()).filter(p -> {
            Mmsi mmsi = (Mmsi) p.id();
            Optional<Info> info = Optional.ofNullable(ships.get(mmsi.value()));
            return (!info.isPresent() || !info.get().shipType.isPresent() || !isTugPilotTowing(info.get().shipType.get()));
        }).filter(p -> p.speedKnots().get() >= 5).compose(CollisionDetector.detectCollisionCandidates()).lift(Logging.<CollisionCandidate>logger().showCount().every(1).showValue().showMemory().log()).doOnNext(cc -> out.format("%s,%s,%s,%s,%s,%s,%s,%s,%s,%s", cc.time(), cc.position1().id().uniqueId(), cc.position1().lat(), cc.position1().lon(), cc.position1().cogDegrees(), cc.position1().speedMetresPerSecond().map(x -> String.valueOf(x)).orElse(""), cc.position2().id().uniqueId(), cc.position2().lat(), cc.position2().lon(), cc.position2().cogDegrees(), cc.position2().speedMetresPerSecond().map(x -> String.valueOf(x)).orElse(""))).count().toBlocking().single();
    }
}