Examples with DogArray org.ddogleg.struct.DogArray used on opensource projects

Example 46 with DogArray

use of org.ddogleg.struct.DogArray in project BoofCV by lessthanoptimal.

the class ExampleSceneRecognition method main.

public static void main(String[] args) {
    String imagePath = UtilIO.pathExample("recognition/scene");
    List<String> images = UtilIO.listByPrefix(imagePath, null, ".jpg");
    Collections.sort(images);
    SceneRecognition<GrayU8> recognizer;
    // Except for real-time applications or when there are more than a few hundred images, you might want to
    // just learn the dictionary from scratch
    var saveDirectory = new File("example_recognition");
    // Tell it to process gray U8 images
    ImageType<GrayU8> imageType = ImageType.SB_U8;
    // Used to look up images one at a time from various sources. In this case a list of images.
    var imageIterator = new ImageFileListIterator<>(images, imageType);
    if (false) {
        // Set the line above to true and it will download a pre-built model. Useful when you have a lot of images
        // or simply want to skip the learning step
        System.out.println("Downloading pre-built model");
        recognizer = RecognitionIO.downloadDefaultSceneRecognition(new File("downloaded_models"), imageType);
        recognizer.setVerbose(System.out, BoofMiscOps.hashSet(BoofVerbose.RECURSIVE));
    } else if (saveDirectory.exists()) {
        System.out.println("Loading previously generated model");
        recognizer = RecognitionIO.loadFeatureToScene(saveDirectory, imageType);
        recognizer.setVerbose(System.out, BoofMiscOps.hashSet(BoofVerbose.RECURSIVE));
    } else {
        // If many applications, learning a new model is a small fraction of the compute time and since its
        // fit to the images it will be more accurate than a generic pre-built model
        System.out.println("Creating a new model");
        var config = new ConfigFeatureToSceneRecognition();
        // Use a hierarchical vocabulary tree, which is very fast and also one of the more accurate approaches
        config.typeRecognize = ConfigFeatureToSceneRecognition.Type.NISTER_2006;
        config.recognizeNister2006.learningMinimumPointsForChildren.setFixed(20);
        recognizer = FactorySceneRecognition.createFeatureToScene(config, imageType);
        // This will print out a lot of debugging information to stdout
        recognizer.setVerbose(System.out, BoofMiscOps.hashSet(BoofVerbose.RECURSIVE));
        // Learn the model from the initial set of images
        recognizer.learnModel(imageIterator);
    }
    // See if the recognition algorithm already has images loaded in to it
    if (recognizer.getImageIds(null).isEmpty()) {
        // Add images to the data base
        System.out.println("Adding images to the database");
        imageIterator.reset();
        while (imageIterator.hasNext()) {
            GrayU8 image = imageIterator.next();
            recognizer.addImage(images.get(imageIterator.getIndex()), image);
        }
        // This saves the model with the image database to disk
        System.out.println("Saving model");
        BoofMiscOps.profile(() -> RecognitionIO.saveFeatureToScene((WrapFeatureToSceneRecognition<GrayU8, ?>) recognizer, saveDirectory), "");
    }
    var gui = new ListDisplayPanel();
    // Specifies which image it will try to look up. In the example, related images are in sets of 3.
    int queryImage = 9;
    // Add the target which the other images are being matched against
    gui.addImage(UtilImageIO.loadImageNotNull(images.get(queryImage)), "Query", ScaleOptions.ALL);
    // Look up images
    var matches = new DogArray<>(SceneRecognition.Match::new);
    recognizer.query(imageIterator.loadImage(queryImage), /* filter */
    (id) -> true, /* limit */
    5, matches);
    for (int i = 0; i < matches.size; i++) {
        String file = matches.get(i).id;
        double error = matches.get(i).error;
        BufferedImage image = UtilImageIO.loadImageNotNull(file);
        String name = FilenameUtils.getBaseName(new File(file).getName());
        gui.addImage(image, String.format("%20s Error %6.3f", name, error), ScaleOptions.ALL);
    }
    System.out.println("Total images = " + images.size());
    System.out.println(images.get(queryImage) + " -> " + matches.get(0).id + " matches.size=" + matches.size);
    ShowImages.showWindow(gui, "Similar Images by Features", true);
}

Example 47 with DogArray

use of org.ddogleg.struct.DogArray in project BoofCV by lessthanoptimal.

the class ExampleMultiBaselineStereo method main.

public static void main(String[] args) {
    // Compute a sparse reconstruction. This will give us intrinsic and extrinsic for all views
    var example = new ExampleMultiViewSparseReconstruction();
    // Specifies the "center" frame to use
    int centerViewIdx = 15;
    example.compute("tree_snow_01.mp4", true);
    // example.compute("ditch_02.mp4", true);
    // example.compute("holiday_display_01.mp4"", true);
    // example.compute("log_building_02.mp4"", true);
    // example.compute("drone_park_01.mp4", false);
    // example.compute("stone_sign.mp4", true);
    // We need a way to load images based on their ID. In this particular case the ID encodes the array index.
    var imageLookup = new LookUpImageFilesByIndex(example.imageFiles);
    // Next we tell it which view to use as the "center", which acts as the common view for all disparity images.
    // The process of selecting the best views to use as centers is a problem all it's own. To keep things
    // we just pick a frame.
    SceneWorkingGraph.View center = example.working.getAllViews().get(centerViewIdx);
    // The final scene refined by bundle adjustment is created by the Working graph. However the 3D relationship
    // between views is contained in the pairwise graph. A View in the working graph has a reference to the view
    // in the pairwise graph. Using that we will find all connected views that have a 3D relationship
    var pairedViewIdxs = new DogArray_I32();
    var sbaIndexToImageID = new TIntObjectHashMap<String>();
    // This relationship between pairwise and working graphs might seem (and is) a bit convoluted. The Pairwise
    // graph is the initial crude sketch of what might be connected. The working graph is an intermediate
    // data structure for computing the metric scene. SBA is a refinement of the working graph.
    // Iterate through all connected views in the pairwise graph and mark their indexes in the working graph
    center.pview.connections.forEach((m) -> {
        // if there isn't a 3D relationship just skip it
        if (!m.is3D)
            return;
        String connectedID = m.other(center.pview).id;
        SceneWorkingGraph.View connected = example.working.views.get(connectedID);
        // Make sure the pairwise view exists in the working graph too
        if (connected == null)
            return;
        // Add this view to the index to name/ID lookup table
        sbaIndexToImageID.put(connected.index, connectedID);
        // Note that this view is one which acts as the second image in the stereo pair
        pairedViewIdxs.add(connected.index);
    });
    // Add the center camera image to the ID look up table
    sbaIndexToImageID.put(centerViewIdx, center.pview.id);
    // Configure there stereo disparity algorithm which is used
    var configDisparity = new ConfigDisparityBMBest5();
    configDisparity.validateRtoL = 1;
    configDisparity.texture = 0.5;
    configDisparity.regionRadiusX = configDisparity.regionRadiusY = 4;
    configDisparity.disparityRange = 120;
    // This is the actual MBS algorithm mentioned previously. It selects the best disparity for each pixel
    // in the original image using a median filter.
    var multiBaseline = new MultiBaselineStereoIndependent<>(imageLookup, ImageType.SB_U8);
    multiBaseline.setStereoDisparity(FactoryStereoDisparity.blockMatchBest5(configDisparity, GrayU8.class, GrayF32.class));
    // Print out verbose debugging and profile information
    multiBaseline.setVerbose(System.out, null);
    multiBaseline.setVerboseProfiling(System.out);
    // Improve stereo by removing small regions, which tends to be noise. Consider adjusting the region size.
    multiBaseline.setDisparitySmoother(FactoryStereoDisparity.removeSpeckle(null, GrayF32.class));
    // Print out debugging information from the smoother
    // Objects.requireNonNull(multiBaseline.getDisparitySmoother()).setVerbose(System.out,null);
    // Creates a list where you can switch between different images/visualizations
    var listDisplay = new ListDisplayPanel();
    listDisplay.setPreferredSize(new Dimension(1000, 300));
    ShowImages.showWindow(listDisplay, "Intermediate Results", true);
    // We will display intermediate results as they come in
    multiBaseline.setListener((leftView, rightView, rectLeft, rectRight, disparity, mask, parameters, rect) -> {
        // Visualize the rectified stereo pair. You can interact with this window and verify
        // that the y-axis is  aligned
        var rectified = new RectifiedPairPanel(true);
        rectified.setImages(ConvertBufferedImage.convertTo(rectLeft, null), ConvertBufferedImage.convertTo(rectRight, null));
        // Cleans up the disparity image by zeroing out pixels that are outside the original image bounds
        RectifyImageOps.applyMask(disparity, mask, 0);
        // Display the colorized disparity
        BufferedImage colorized = VisualizeImageData.disparity(disparity, null, parameters.disparityRange, 0);
        SwingUtilities.invokeLater(() -> {
            listDisplay.addItem(rectified, "Rectified " + leftView + " " + rightView);
            listDisplay.addImage(colorized, leftView + " " + rightView);
        });
    });
    // Process the images and compute a single combined disparity image
    if (!multiBaseline.process(example.scene, center.index, pairedViewIdxs, sbaIndexToImageID::get)) {
        throw new RuntimeException("Failed to fuse stereo views");
    }
    // Extract the point cloud from the fused disparity image
    GrayF32 fusedDisparity = multiBaseline.getFusedDisparity();
    DisparityParameters fusedParam = multiBaseline.getFusedParam();
    BufferedImage colorizedDisp = VisualizeImageData.disparity(fusedDisparity, null, fusedParam.disparityRange, 0);
    ShowImages.showWindow(colorizedDisp, "Fused Disparity");
    // Now compute the point cloud it represents and the color of each pixel.
    // For the fused image, instead of being in rectified image coordinates it's in the original image coordinates
    // this makes extracting color much easier.
    var cloud = new DogArray<>(Point3D_F64::new);
    var cloudRgb = new DogArray_I32(cloud.size);
    // Load the center image in color
    var colorImage = new InterleavedU8(1, 1, 3);
    imageLookup.loadImage(center.pview.id, colorImage);
    // Since the fused image is in the original (i.e. distorted) pixel coordinates and is not rectified,
    // that needs to be taken in account by undistorting the image to create the point cloud.
    CameraPinholeBrown intrinsic = BundleAdjustmentOps.convert(example.scene.cameras.get(center.cameraIdx).model, colorImage.width, colorImage.height, null);
    Point2Transform2_F64 pixel_to_norm = new LensDistortionBrown(intrinsic).distort_F64(true, false);
    MultiViewStereoOps.disparityToCloud(fusedDisparity, fusedParam, new PointToPixelTransform_F64(pixel_to_norm), (pixX, pixY, x, y, z) -> {
        cloud.grow().setTo(x, y, z);
        cloudRgb.add(colorImage.get24(pixX, pixY));
    });
    // Configure the point cloud viewer
    PointCloudViewer pcv = VisualizeData.createPointCloudViewer();
    pcv.setCameraHFov(UtilAngle.radian(70));
    pcv.setTranslationStep(0.15);
    pcv.addCloud(cloud.toList(), cloudRgb.data);
    // pcv.setColorizer(new SingleAxisRgb.Z().fperiod(30.0));
    JComponent viewer = pcv.getComponent();
    viewer.setPreferredSize(new Dimension(600, 600));
    ShowImages.showWindow(viewer, "Point Cloud", true);
    System.out.println("Done");
}

Example 48 with DogArray

use of org.ddogleg.struct.DogArray in project BoofCV by lessthanoptimal.

the class ExampleDepthPointCloud method main.

public static void main(String[] args) {
    String nameRgb = UtilIO.pathExample("kinect/basket/basket_rgb.png");
    String nameDepth = UtilIO.pathExample("kinect/basket/basket_depth.png");
    String nameCalib = UtilIO.pathExample("kinect/basket/visualdepth.yaml");
    VisualDepthParameters param = CalibrationIO.load(nameCalib);
    BufferedImage buffered = UtilImageIO.loadImageNotNull(nameRgb);
    Planar<GrayU8> rgb = ConvertBufferedImage.convertFromPlanar(buffered, null, true, GrayU8.class);
    GrayU16 depth = ConvertBufferedImage.convertFrom(UtilImageIO.loadImageNotNull(nameDepth), null, GrayU16.class);
    var cloud = new DogArray<>(Point3D_F64::new);
    var cloudColor = new DogArray<>(() -> new int[3]);
    VisualDepthOps.depthTo3D(param.visualParam, rgb, depth, cloud, cloudColor);
    PointCloudViewer viewer = VisualizeData.createPointCloudViewer();
    viewer.setCameraHFov(PerspectiveOps.computeHFov(param.visualParam));
    viewer.setTranslationStep(15);
    for (int i = 0; i < cloud.size; i++) {
        Point3D_F64 p = cloud.get(i);
        int[] color = cloudColor.get(i);
        int c = (color[0] << 16) | (color[1] << 8) | color[2];
        viewer.addPoint(p.x, p.y, p.z, c);
    }
    viewer.getComponent().setPreferredSize(new Dimension(rgb.width, rgb.height));
    // ---------- Display depth image
    // use the actual max value in the image to maximize its appearance
    int maxValue = ImageStatistics.max(depth);
    BufferedImage depthOut = VisualizeImageData.disparity(depth, null, maxValue, 0);
    ShowImages.showWindow(depthOut, "Depth Image", true);
    // ---------- Display colorized point cloud
    ShowImages.showWindow(viewer.getComponent(), "Point Cloud", true);
    System.out.println("Total points = " + cloud.size);
}

Example 49 with DogArray

use of org.ddogleg.struct.DogArray in project BoofCV by lessthanoptimal.

the class MultiViewOps method triangulatePoints.

/**
 * Convenience function for initializing bundle adjustment parameters. Triangulates points using camera
 * position and pixel observations.
 *
 * @param structure camera locations
 * @param observations observations of features in the images
 */
public static void triangulatePoints(SceneStructureMetric structure, SceneObservations observations) {
    TriangulateNViewsMetricH triangulator = FactoryMultiView.triangulateNViewMetricH(ConfigTriangulation.GEOMETRIC());
    List<RemoveBrownPtoN_F64> list_p_to_n = new ArrayList<>();
    for (int i = 0; i < structure.cameras.size; i++) {
        RemoveBrownPtoN_F64 p2n = new RemoveBrownPtoN_F64();
        BundleAdjustmentCamera baseModel = Objects.requireNonNull(structure.cameras.data[i].model);
        if (baseModel instanceof BundlePinholeSimplified) {
            BundlePinholeSimplified cam = (BundlePinholeSimplified) baseModel;
            p2n.setK(cam.f, cam.f, 0, 0, 0).setDistortion(new double[] { cam.k1, cam.k2 }, 0, 0);
        } else if (baseModel instanceof BundlePinhole) {
            BundlePinhole cam = (BundlePinhole) baseModel;
            p2n.setK(cam.fx, cam.fy, cam.skew, cam.cx, cam.cy).setDistortion(new double[] { 0, 0 }, 0, 0);
        } else if (baseModel instanceof BundlePinholeBrown) {
            BundlePinholeBrown cam = (BundlePinholeBrown) baseModel;
            p2n.setK(cam.fx, cam.fy, cam.skew, cam.cx, cam.cy).setDistortion(cam.radial, cam.t1, cam.t2);
        } else {
            throw new RuntimeException("Unknown camera model!");
        }
        list_p_to_n.add(p2n);
    }
    DogArray<Point2D_F64> normObs = new DogArray<>(Point2D_F64::new);
    normObs.resize(3);
    final boolean homogenous = structure.isHomogenous();
    Point4D_F64 X = new Point4D_F64();
    List<Se3_F64> worldToViews = new ArrayList<>();
    for (int i = 0; i < structure.points.size; i++) {
        normObs.reset();
        worldToViews.clear();
        SceneStructureCommon.Point sp = structure.points.get(i);
        for (int j = 0; j < sp.views.size; j++) {
            int viewIdx = sp.views.get(j);
            SceneStructureMetric.View v = structure.views.data[viewIdx];
            worldToViews.add(structure.getParentToView(v));
            // get the observation in pixels
            Point2D_F64 n = normObs.grow();
            int pointidx = observations.views.get(viewIdx).point.indexOf(i);
            observations.views.get(viewIdx).getPixel(pointidx, n);
            // convert to normalized image coordinates
            list_p_to_n.get(v.camera).compute(n.x, n.y, n);
        }
        if (!triangulator.triangulate(normObs.toList(), worldToViews, X)) {
            // this should work unless the input is bad
            throw new RuntimeException("Triangulation failed. Bad input?");
        }
        if (homogenous)
            sp.set(X.x, X.y, X.z, X.w);
        else
            sp.set(X.x / X.w, X.y / X.w, X.z / X.w);
    }
}

Example 50 with DogArray

use of org.ddogleg.struct.DogArray in project BoofCV by lessthanoptimal.

the class TestMultiViewIO method save_load_SimilarImages.

@Test
void save_load_SimilarImages() {
    SimilarImagesData expected = new SimilarImagesData();
    for (int i = 0; i < 4; i++) {
        String id = "" + i;
        List<Point2D_F64> features = new ArrayList<>();
        for (int j = 0; j < 4 + i; j++) {
            features.add(new Point2D_F64(i + j, 1));
        }
        expected.add(id, features);
    }
    var matches12 = new ArrayList<AssociatedIndex>();
    for (int i = 0; i < 8; i++) {
        matches12.add(new AssociatedIndex(rand.nextInt(), rand.nextInt()));
    }
    expected.setRelationship("2", "1", matches12);
    var output = new ByteArrayOutputStream();
    MultiViewIO.save(expected, new OutputStreamWriter(output, UTF_8));
    var input = new ByteArrayInputStream(output.toByteArray());
    LookUpSimilarImages found = MultiViewIO.loadSimilarImages(new InputStreamReader(input, UTF_8));
    assertEquals(expected.listImages.size(), found.getImageIDs().size());
    DogArray<Point2D_F64> features = new DogArray<>(Point2D_F64::new);
    DogArray<AssociatedIndex> pairs = new DogArray<>(AssociatedIndex::new);
    for (String id : expected.listImages) {
        int i = Integer.parseInt(id);
        found.lookupPixelFeats(id, features);
        assertEquals(4 + i, features.size);
    }
    for (String id : expected.listImages) {
        int i = Integer.parseInt(id);
        List<String> similarIds = new ArrayList<>();
        found.findSimilar(id, (s) -> true, similarIds);
        if (i != 2 && i != 1) {
            assertEquals(0, similarIds.size());
            continue;
        }
        assertEquals(1, similarIds.size());
        found.lookupAssociated(similarIds.get(0), pairs);
        assertEquals(8, pairs.size());
    }
}