Examples with Landmark ai.djl.modality.cv.output.Landmark used on opensource projects

Example 1 with Landmark

use of ai.djl.modality.cv.output.Landmark in project djl by deepjavalibrary.

the class OpenCVImageFactoryTest method testImage.

@Test
public void testImage() throws IOException {
    // failed on Windows ServerCore container
    TestRequirements.notWindows();
    ImageFactory factory = ImageFactory.getInstance();
    ImageFactory defFactory = new BufferedImageFactory();
    Path path = Paths.get("../../examples/src/test/resources/kitten.jpg");
    String url = path.toUri().toURL().toString();
    Image gold = defFactory.fromFile(path);
    try (NDManager manager = NDManager.newBaseManager()) {
        NDArray expected = gold.toNDArray(manager);
        Image img = factory.fromFile(path);
        Assert.assertEquals(img.getWidth(), gold.getWidth());
        Assert.assertEquals(img.getHeight(), gold.getHeight());
        Assert.assertEquals(img.getWrappedImage().getClass().getSimpleName(), "Mat");
        NDArray array = img.toNDArray(manager);
        Assert.assertEquals(array, expected);
        Image newImage = factory.fromNDArray(array);
        Assert.assertEquals(newImage.getWidth(), gold.getWidth());
        Assert.assertEquals(newImage.getHeight(), gold.getHeight());
        array = array.transpose(2, 0, 1);
        newImage = factory.fromNDArray(array);
        Assert.assertEquals(newImage.getWidth(), gold.getWidth());
        Assert.assertEquals(newImage.getHeight(), gold.getHeight());
        img = factory.fromUrl(url);
        array = img.toNDArray(manager);
        Assert.assertEquals(array, expected);
        newImage = factory.fromImage(img.getWrappedImage());
        newImage = newImage.getSubImage(0, 0, 4, 2);
        array = newImage.toNDArray(manager, Image.Flag.GRAYSCALE);
        expected = gold.getSubImage(0, 0, 4, 2).toNDArray(manager, Image.Flag.GRAYSCALE);
        Assert.assertEquals(array, expected);
        List<String> list = Collections.singletonList("cat");
        List<Double> prob = Collections.singletonList(0.9);
        List<ai.djl.modality.cv.output.Point> points = new ArrayList<>();
        points.add(new ai.djl.modality.cv.output.Point(120, 160));
        List<BoundingBox> boundingBoxes = Collections.singletonList(new Landmark(0.35, 0.15, 0.5, 0.6, points));
        DetectedObjects detectedObjects = new DetectedObjects(list, prob, boundingBoxes);
        Image imgCopy = img.duplicate();
        imgCopy.drawBoundingBoxes(detectedObjects);
        float[][] maskProb = { { 1f, 1f, 1f, 1f, 1f, 1f, 1f }, { 1f, 1f, 1f, 1f, 1f, 1f, 1f }, { 1f, 1f, 1f, 1f, 1f, 1f, 1f }, { 1f, 1f, 1f, 1f, 1f, 1f, 1f }, { 1f, 1f, 1f, 1f, 1f, 1f, 1f }, { 1f, 1f, 1f, 1f, 1f, 1f, 1f } };
        List<BoundingBox> masks = Collections.singletonList(new Mask(0.1, 0.5, 0.1, 0.1, maskProb));
        DetectedObjects mask = new DetectedObjects(list, prob, masks);
        imgCopy.drawBoundingBoxes(mask);
        List<Joints.Joint> jointList = Collections.singletonList(new Joints.Joint(0.2, 0.2, 0.8));
        Joints joints = new Joints(jointList);
        imgCopy.drawJoints(joints);
        try (OutputStream os = Files.newOutputStream(Paths.get("build/newImage.png"))) {
            imgCopy.save(os, "png");
        }
        Assert.assertThrows(IOException.class, () -> {
            factory.fromFile(Paths.get("nonexist.jpg"));
        });
        Assert.assertThrows(IOException.class, () -> {
            factory.fromUrl("file:build.gradle");
        });
    }
}

Example 2 with Landmark

use of ai.djl.modality.cv.output.Landmark in project djl by deepjavalibrary.

the class FaceDetectionTranslator method processOutput.

/**
 * {@inheritDoc}
 */
@Override
public DetectedObjects processOutput(TranslatorContext ctx, NDList list) {
    NDManager manager = ctx.getNDManager();
    double scaleXY = variance[0];
    double scaleWH = variance[1];
    NDArray prob = list.get(1).get(":, 1:");
    prob = NDArrays.stack(new NDList(prob.argMax(1).toType(DataType.FLOAT32, false), prob.max(new int[] { 1 })));
    NDArray boxRecover = boxRecover(manager, width, height, scales, steps);
    NDArray boundingBoxes = list.get(0);
    NDArray bbWH = boundingBoxes.get(":, 2:").mul(scaleWH).exp().mul(boxRecover.get(":, 2:"));
    NDArray bbXY = boundingBoxes.get(":, :2").mul(scaleXY).mul(boxRecover.get(":, 2:")).add(boxRecover.get(":, :2")).sub(bbWH.mul(0.5f));
    boundingBoxes = NDArrays.concat(new NDList(bbXY, bbWH), 1);
    NDArray landms = list.get(2);
    landms = decodeLandm(landms, boxRecover, scaleXY);
    // filter the result below the threshold
    NDArray cutOff = prob.get(1).gt(confThresh);
    boundingBoxes = boundingBoxes.transpose().booleanMask(cutOff, 1).transpose();
    landms = landms.transpose().booleanMask(cutOff, 1).transpose();
    prob = prob.booleanMask(cutOff, 1);
    // start categorical filtering
    long[] order = prob.get(1).argSort().get(":" + topK).toLongArray();
    prob = prob.transpose();
    List<String> retNames = new ArrayList<>();
    List<Double> retProbs = new ArrayList<>();
    List<BoundingBox> retBB = new ArrayList<>();
    Map<Integer, List<BoundingBox>> recorder = new ConcurrentHashMap<>();
    for (int i = order.length - 1; i >= 0; i--) {
        long currMaxLoc = order[i];
        float[] classProb = prob.get(currMaxLoc).toFloatArray();
        int classId = (int) classProb[0];
        double probability = classProb[1];
        double[] boxArr = boundingBoxes.get(currMaxLoc).toDoubleArray();
        double[] landmsArr = landms.get(currMaxLoc).toDoubleArray();
        Rectangle rect = new Rectangle(boxArr[0], boxArr[1], boxArr[2], boxArr[3]);
        List<BoundingBox> boxes = recorder.getOrDefault(classId, new ArrayList<>());
        boolean belowIoU = true;
        for (BoundingBox box : boxes) {
            if (box.getIoU(rect) > nmsThresh) {
                belowIoU = false;
                break;
            }
        }
        if (belowIoU) {
            List<Point> keyPoints = new ArrayList<>();
            for (int j = 0; j < 5; j++) {
                // 5 face landmarks
                double x = landmsArr[j * 2];
                double y = landmsArr[j * 2 + 1];
                keyPoints.add(new Point(x * width, y * height));
            }
            Landmark landmark = new Landmark(boxArr[0], boxArr[1], boxArr[2], boxArr[3], keyPoints);
            boxes.add(landmark);
            recorder.put(classId, boxes);
            // classes.get(classId)
            String className = "Face";
            retNames.add(className);
            retProbs.add(probability);
            retBB.add(landmark);
        }
    }
    return new DetectedObjects(retNames, retProbs, retBB);
}

Example 3 with Landmark

use of ai.djl.modality.cv.output.Landmark in project djl by deepjavalibrary.

the class OpenCVImage method drawBoundingBoxes.

/**
 * {@inheritDoc}
 */
@Override
public void drawBoundingBoxes(DetectedObjects detections) {
    int imageWidth = image.width();
    int imageHeight = image.height();
    List<DetectedObjects.DetectedObject> list = detections.items();
    for (DetectedObjects.DetectedObject result : list) {
        String className = result.getClassName();
        BoundingBox box = result.getBoundingBox();
        Rectangle rectangle = box.getBounds();
        int x = (int) (rectangle.getX() * imageWidth);
        int y = (int) (rectangle.getY() * imageHeight);
        Rect rect = new Rect(x, y, (int) (rectangle.getWidth() * imageWidth), (int) (rectangle.getHeight() * imageHeight));
        Scalar color = new Scalar(RandomUtils.nextInt(178), RandomUtils.nextInt(178), RandomUtils.nextInt(178));
        Imgproc.rectangle(image, rect.tl(), rect.br(), color, 2);
        Size size = Imgproc.getTextSize(className, Imgproc.FONT_HERSHEY_PLAIN, 1.3, 1, null);
        Point br = new Point(x + size.width + 4, y + size.height + 4);
        Imgproc.rectangle(image, rect.tl(), br, color, -1);
        Point point = new Point(x, y + size.height + 2);
        color = new Scalar(255, 255, 255);
        Imgproc.putText(image, className, point, Imgproc.FONT_HERSHEY_PLAIN, 1.3, color, 1);
        // If we have a mask instead of a plain rectangle, draw tha mask
        if (box instanceof Mask) {
            Mask mask = (Mask) box;
            BufferedImage img = mat2Image(image);
            drawMask(img, mask);
            image = image2Mat(img);
        } else if (box instanceof Landmark) {
            drawLandmarks(box);
        }
    }
}