We Keep Coding

Opencv detection Java

I will like to detect the hand in the following images. I have tried grabCut algorithm from opencv using the foreground and background extraction. However, there is some inaccuracy in the detection. Could someone help me with it, please. Thank You.
The code is as follow:
public static void DetectionAlg() {
//convert from bitmap to mat
Mat mat = new Mat(myBitmap.getWidth(), myBitmap.getHeight(), CvType.CV_8UC3);
Utils.bitmapToMat(myBitmap, mat);
//covert to bgr
Mat bgrMat = new Mat();
Imgproc.cvtColor(mat, bgrMat, Imgproc.COLOR_BGRA2BGR);
Mat background = new Mat(bgrMat.size(), CvType.CV_8UC3, new Scalar(255, 255, 255));
Mat firstMask = new Mat();
Mat bgModel = new Mat();
Mat fgModel = new Mat();
Mat mask;
Mat source = new Mat(1, 1, CvType.CV_8U, new Scalar(Imgproc.GC_PR_FGD));
Mat dst = new Mat();
//using a user define area
Rect rect = new Rect(GlobalVar.tl, GlobalVar.br);
Imgproc.grabCut(bgrMat, firstMask, rect, bgModel, fgModel, 5, Imgproc.GC_INIT_WITH_RECT);
Core.compare(firstMask, source, firstMask, Core.CMP_EQ);
Mat foreground = new Mat(bgrMat.size(), CvType.CV_8UC3, new Scalar(255, 255, 255));
bgrMat.copyTo(foreground, firstMask);
Scalar color = new Scalar(255, 0, 0, 255);
Imgproc.rectangle(bgrMat, GlobalVar.tl, GlobalVar.br, color);
Mat tmp = new Mat();
Imgproc.resize(background, tmp, bgrMat.size());
background = tmp;
mask = new Mat(foreground.size(), CvType.CV_8UC1, new Scalar(255, 255, 255));
Imgproc.cvtColor(foreground, mask, Imgproc.COLOR_BGR2GRAY);
Imgproc.threshold(mask, mask, 254, 255, Imgproc.THRESH_BINARY_INV);
Mat vals = new Mat(1, 1, CvType.CV_8UC3, new Scalar(0.0));
background.copyTo(dst);
background.setTo(vals, mask);
Core.add(background, foreground, dst, mask);
firstMask.release();
source.release();
bgModel.release();
fgModel.release();
vals.release();
Mat colorMat = new Mat();
Imgproc.cvtColor(dst, colorMat, Imgproc.COLOR_BGR2BGRA);
Bitmap outputBitmap = Bitmap.createBitmap(colorMat.cols(), colorMat.rows(), Bitmap.Config.ARGB_8888);
Utils.matToBitmap(colorMat, outputBitmap);
}
Images are in the link:
This is the input image
This is another input image
This is the output using the above method, however, there are some other parts that does not belong to the hand

OpenCV Java - Changing pixel color

I am trying to determine a way to change the pixel color of my masks from black to a different color. Unfortunately, I have not be able to determine a way to do this task. Essentially, what I am trying to do is take this image:
and convert the black portions to a color with values (255, 160, 130). I have tried several methods to try and achieve my goal. These include draw contours, setTo, and looping through the matrix. Unfortunately all of these attempts have failed. I have included the code and the resulting outcomes below.
Draw Contours method
System.loadLibrary(Core.NATIVE_LIBRARY_NAME);
Mat img = Imgcodecs.imread(
"C:\\Users\\Hassan\\Documents\\School\\Me\\COMP5900 Y\\Project\\Project\\src\\resources\\face.jpg");
Mat img_grey = new Mat();
Mat grad = new Mat(), grad_x = new Mat(), grad_y = new Mat();
Mat abs_grad_x = new Mat(), abs_grad_y = new Mat();
int ddepth = CvType.CV_32F;
int scale = 1;
int delta = 0;
Imgproc.GaussianBlur(img, img, new Size(3, 3), 0, 0, Core.BORDER_CONSTANT);
Imgproc.cvtColor(img, img_grey, Imgproc.COLOR_BGR2GRAY);
// Apply Sobel
Imgproc.Sobel(img_grey, grad_x, ddepth, 1, 0, 3, scale, delta, Core.BORDER_DEFAULT);
Imgproc.Sobel(img_grey, grad_y, ddepth, 0, 1, 3, scale, delta, Core.BORDER_DEFAULT);
// converting back to CV_8U
Core.convertScaleAbs(grad_x, abs_grad_x);
Core.convertScaleAbs(grad_y, abs_grad_y);
// Total Gradient (approximate)
Core.addWeighted(abs_grad_x, 0.5, abs_grad_y, 0.5, 0, grad);
Photo.fastNlMeansDenoising(grad, grad);
Imgproc.GaussianBlur(grad, grad, new Size(3, 3), 0, 0, Core.BORDER_CONSTANT);
// isolate background
Mat background = new Mat();
Imgproc.threshold(grad, background, 2, 255, Imgproc.THRESH_BINARY);
// draw contours
List<MatOfPoint> contours = new ArrayList<>();
Mat hierarchy = new Mat();
Imgproc.findContours(background, contours, hierarchy, Imgproc.RETR_TREE, Imgproc.CHAIN_APPROX_NONE);
Mat drawing = Mat.zeros(background.size(), CvType.CV_8UC3);
List<MatOfPoint> hullList = new ArrayList<>();
for (MatOfPoint contour : contours) {
MatOfInt hull = new MatOfInt();
Imgproc.convexHull(contour, hull);
Point[] contourArray = contour.toArray();
Point[] hullPoints = new Point[hull.rows()];
List<Integer> hullContourIdxList = hull.toList();
for (int i = 0; i < hullContourIdxList.size(); i++) {
hullPoints[i] = contourArray[hullContourIdxList.get(i)];
}
hullList.add(new MatOfPoint(hullPoints));
}
for (int i = 0; i < contours.size(); i++) {
Scalar color = new Scalar(255, 160, 130);
Imgproc.drawContours(drawing, contours, i, color);
//Imgproc.drawContours(drawing, hullList, i, color );
}
Note here, that I also tried using Imgproc.RETR_EXTERNAL as well, but that produced a completely black image. Also the name of the HighGui window is called "flood fill", but I just forgot to update the name.
setTo
// replace find and draw contours portion of code above
Mat out = new Mat();
background.copyTo(out);
out.setTo(new Scalar(255, 160, 130), background);
Iterating through matrix
// replace draw contours portion of code above
for (a = 0; a < background.rows(); a++) {
for(b = 0; b < background.cols(); b++) {
if(background.get(a,b)[0] == 0) {
//background.put(a, b, CvType.CV_16F, new Scalar(255, 160, 130));
double[] data = {255, 160, 130};
background.put(a, b, data);
}
}
}
The loop is promising, but I know it will not be efficient as I have 2 other masks that I would like to update as well. Could you please suggest an efficient method, that allows me to set the value for all three channels?
Thanks

I am not sure why you are doing many operations on the image but to me it looks like applying the mask and replacing the color efficiently. So if there are other complexities than please let me know.
Below is the code I was looking for in Java.
public static void main(String s[]) {
System.loadLibrary(Core.NATIVE_LIBRARY_NAME);
Mat matr =Imgcodecs.imread("/home/shariq/Desktop/test.png");
Mat result = new Mat();
//create a mask based on range
Core.inRange(matr, new Scalar(0), new Scalar(50), result);
Imgcodecs.imwrite("/home/shariq/Desktop/test_in.png", result);
//apply the mask with color you are looking for, note here scalar is in hsv
matr.setTo(new Scalar(130,160,255),result);
Imgcodecs.imwrite("/home/shariq/Desktop/result.png", matr);
}
We are creating a mask for the pixel values between 0-50 for black color using inRange method.
Core.inRange(matr, new Scalar(0), new Scalar(50), result);
This mask in result variable is than applied to original matrix using setTo method. The replacement color value is provided in HSV format through Scalar object. new Scalar(a,b,c) in HSV can be understand in RGB like this Red = c, Green = b and Blue = a.
matr.setTo(new Scalar(130,160,255),result);
Its quite fast compared to iterating the pixels one by one.

How to draw a boundingRect with the right rotation angle by using OpenCV?

I am performing Canny edge detector using Android and Opencv on an image to detect the largest contour, extract it using warpPerspective method then find all objects inside that contour. Everything is working as expected but only for image that isn't rotated.
I am using boundingRect to get the contour and use its coordinates to extract it.
Here my code:
private Mat detectLargestContour(Mat origMat) {
// long e1 = Core.getTickCount();
Mat mGray = new Mat();
MatOfDouble mu = new MatOfDouble();
MatOfDouble stddev = new MatOfDouble();
Imgproc.cvtColor(origMat, mGray, Imgproc.COLOR_BGR2GRAY);
Core.meanStdDev(mGray, mu, stddev);
Imgproc.GaussianBlur(mGray, mGray, new Size(5, 5), 5);
//Imgproc.Canny(mGray, mGray, 30, 80, 3, false); //FOR HIGH BRIGHTNESS
//Imgproc.Canny(mGray, mGray, 50, 130, 3, false); // FOR LOW BRIGHTNESS
Imgproc.Canny(mGray, mGray, mu.get(0, 0)[0], stddev.get(0, 0)[0], 3, false);
Mat kernell = Imgproc.getStructuringElement(Imgproc.MORPH_RECT, new Size(9,9));
Imgproc.morphologyEx(mGray, mGray, Imgproc.MORPH_CLOSE, kernell);
Imgproc.dilate(mGray, mGray, Imgproc.getStructuringElement(Imgproc.MORPH_CROSS, new Size(3, 3)));
List<MatOfPoint> contours = new ArrayList<MatOfPoint>();
Mat hierarchy = new Mat();
Imgproc.findContours(mGray, contours, hierarchy, Imgproc.RETR_EXTERNAL, Imgproc.CHAIN_APPROX_SIMPLE);
//MatOfPoint2f approxCurve = new MatOfPoint2f();
double largest_area=0;
Rect rect = new Rect();
for (int idx = 0; idx < contours.size() ; idx++) {
double a = Imgproc.contourArea(contours.get(idx)); //Find the area of contour
if (a > largest_area) {
largest_area = a;
rect = Imgproc.boundingRect(contours.get(idx));
}
}
if (rect.area() > 100000) {
Imgproc.rectangle(origMat, rect.tl(), rect.br(), new Scalar(0, 255, 0));
p1 = new Point(rect.tl().x, rect.tl().y);
p2 = new Point(rect.tl().x + rect.width, rect.tl().y);
p3 = new Point(rect.tl().x, rect.tl().y + rect.height);
p4 = new Point(rect.tl().x + rect.width, rect.tl().y + rect.height);
card_corners = new ArrayList<>();
card_corners.add(p1);
card_corners.add(p3);
card_corners.add(p4);
card_corners.add(p2);
warpedCard = new Mat(origMat.rows(), origMat.cols(), CvType.CV_8UC3);
final Point p1 = new Point(warpedCard.cols() + marge, warpedCard.rows() + marge);
final Point p2 = new Point(0 - marge, warpedCard.rows() + marge);
final Point p3 = new Point(0 - marge, 0 - marge);
final Point p4 = new Point(warpedCard.cols() + marge, 0 - marge);
LinkedList<Point> sceneList = new LinkedList<Point>();
sceneList.addLast(p4);
sceneList.addLast(p3);
sceneList.addLast(p2);
sceneList.addLast(p1);
MatOfPoint2f scene = new MatOfPoint2f();
scene.fromList(sceneList);
MatOfPoint2f obj = new MatOfPoint2f();
obj.fromList(card_corners);
Mat homography = Calib3d.findHomography(obj, scene);
Imgproc.warpPerspective(origMat, warpedCard, homography, new Size(warpedCard.cols(), warpedCard.rows()));
return warpedCard;
}
return origMat;
}
It's weird but only boundingRect gave me a stable and performant result but the drawn rectangle doesn't rotate with the found contour.
How can I manage to resolve this issue? Any thoughts?
EDIT:
I changed boundingRect with minAreaRect.
Here is the code
int largest_idx = 0;
for (int idx = 0; idx < contours.size() ; idx++) {
double a = Imgproc.contourArea(contours.get(idx)); //Find the area of contour
if (a > largest_area) {
largest_area = a;
// rect = Imgproc.boundingRect(contours.get(idx));
largest_idx = idx;
}
}
MatOfPoint2f new_mat = new MatOfPoint2f( contours.get(largest_idx).toArray() );
RotatedRect rbox = Imgproc.minAreaRect(new_mat);
Log.d("rotatedrect_angle", "" + rbox.angle);
Point points[] = new Point[4];
rbox.points(points);
for(int i=0; i<4; ++i){
Imgproc.line(origMat, points[i], points[(i+1)%4], new Scalar(255,255,255));
}
And here is what I've got:
As you can see the detection isn't as accurate as when I used boundingRect.

A Python demo to find and draw a rotatedRect:
# 2019/03/01
# https://stackoverflow.com/a/54942835/3547485
import numpy as np
import cv2
gray = cv2.imread("tmp.png", cv2.IMREAD_GRAYSCALE)
th, threshed = cv2.threshold(gray, 220, 255, cv2.THRESH_BINARY_INV)
cnts = cv2.findContours(threshed, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)[-2]
cnt = sorted(cnts, key=cv2.contourArea, reverse=True)[0]
rbox = cv2.minAreaRect(cnt)
pts = cv2.boxPoints(rbox).astype(np.int32)
cv2.drawContours(img, [pts], -1, (0, 255, 0), 1, cv2.LINE_AA)
cv2.imwrite("dst.png", img)
Useful OpenCV functions(in Python) : cv2.minAreaRect, cv2.boxPoints, cv.2drawContours. You can find corresponding functions in Java.

how to extract an object from the camera preview android opencv

I am trying to scan a MTG card using OpenCV on android. I have it to where I can detect the edges of the card and even draw an outline around it but am confused on how to extract just the card from the background and then exit from the camera preview. Here is my code so far:
public Mat onCameraFrame(CameraBridgeViewBase.CvCameraViewFrame inputFrame) {
Mat result = new Mat();
Mat mask = new Mat();
List<MatOfPoint> contours = new ArrayList<>();
Mat hierarchy = new Mat();
mRgba = inputFrame.rgba();
Imgproc.Canny(mRgba, result, 40, 120);
Imgproc.GaussianBlur(result, result, new Size(9,9), 2, 2);
Imgproc.findContours(result, contours, hierarchy, Imgproc.RETR_EXTERNAL, Imgproc.CHAIN_APPROX_SIMPLE, new Point(0,0));
Imgproc.drawContours(mask, contours, -1, new Scalar(0, 255, 0), 1);
hierarchy.release();
for ( int contourIdx=0; contourIdx < contours.size(); contourIdx++ )
{
// Minimum size allowed for consideration
MatOfPoint2f approxCurve = new MatOfPoint2f();
MatOfPoint2f contour2f = new MatOfPoint2f( contours.get(contourIdx).toArray() );
//Processing on mMOP2f1 which is in type MatOfPoint2f
double approxDistance = Imgproc.arcLength(contour2f, true)*0.02;
Imgproc.approxPolyDP(contour2f, approxCurve, approxDistance, true);
//Convert back to MatOfPoint
MatOfPoint points = new MatOfPoint( approxCurve.toArray() );
// Get bounding rect of contour
Rect rect = Imgproc.boundingRect(points);
Imgproc.rectangle(mRgba, new Point(rect.x, rect.y), new Point(rect.x + rect.width, rect.y + rect.height), new Scalar(255, 0, 0, 255), 3);
}
Bitmap card = Bitmap.createBitmap(result.cols(), result.rows(), Bitmap.Config.ARGB_8888);
Utils.matToBitmap(result, card);
return mRgba;
}
}
Here is an example of what it looks like when run the code. As you can see the card is outlined by a red rectangle now but how do I extract just whats in the rectangle, save it to a mat or bitmap and then exit the camera?
screenshot of outlined card

I didn't test it, but can't you just use...
return new Mat(mRgba, rect);

how to use opencv to calculate HSV histogram in java platform?

Actually, i want to find out the dominant colour in the image, so i want to find the HSV histogram of the image and hence filter out the other colours. However, i dont know how to do this in java platform using opence. I only find the code in C++. Thank you.
Mat image = Highgui.imread("binary07.jpg");
//Mat src = new Mat(image.height(), image.width(), CvType.CV_8UC2);
Imgproc.cvtColor(image, image, Imgproc.COLOR_RGB2GRAY);
List<Mat> hsv_planes = new ArrayList<Mat>();
Core.split(image, hsv_planes);
MatOfInt histSize = new MatOfInt(256);
final MatOfFloat histRange = new MatOfFloat(0f, 256f);
boolean accumulate = false;
Mat h_hist = new Mat();
Mat s_hist = new Mat();
Mat v_hist = new Mat();
//error appear in the following sentences
Imgproc.calcHist((List<Mat>) hsv_planes.get(0), new MatOfInt(3), new Mat(), h_hist, histSize, histRange, accumulate);
Imgproc.calcHist((List<Mat>) hsv_planes.get(1), new MatOfInt(3), new Mat(), s_hist, histSize, histRange, accumulate);
Imgproc.calcHist((List<Mat>) hsv_planes.get(2), new MatOfInt(3), new Mat(), v_hist, histSize, histRange, accumulate);
int hist_w = 512;
int hist_h = 600;
long bin_w = Math.round((double) hist_w / 256);
//bin_w = Math.round((double) (hist_w / 256));
Mat histImage = new Mat(hist_h, hist_w, CvType.CV_8UC1);
Core.normalize(h_hist, h_hist, 3, histImage.rows(), Core.NORM_MINMAX);
Core.normalize(s_hist, s_hist, 3, histImage.rows(), Core.NORM_MINMAX);
Core.normalize(v_hist, v_hist, 3, histImage.rows(), Core.NORM_MINMAX);
for (int i = 1; i < 256; i++) {
Point p1 = new Point(bin_w * (i - 1), hist_h - Math.round(h_hist.get(i - 1, 0)[0]));
Point p2 = new Point(bin_w * (i), hist_h - Math.round(h_hist.get(i, 0)[0]));
Core.line(histImage, p1, p2, new Scalar(255, 0, 0), 2, 8, 0);
Point p3 = new Point(bin_w * (i - 1), hist_h - Math.round(s_hist.get(i - 1, 0)[0]));
Point p4 = new Point(bin_w * (i), hist_h - Math.round(s_hist.get(i, 0)[0]));
Core.line(histImage, p3, p4, new Scalar(0, 255, 0), 2, 8, 0);
Point p5 = new Point(bin_w * (i - 1), hist_h - Math.round(v_hist.get(i - 1, 0)[0]));
Point p6 = new Point(bin_w * (i), hist_h - Math.round(v_hist.get(i, 0)[0]));
Core.line(histImage, p5, p6, new Scalar(0, 0, 255), 2, 8, 0);
}
Highgui.imwrite("histogram.jpg", histImage);
I dont know how to get the output after the split function.
reference:
http://docs.opencv.org/java/
http://docs.opencv.org/doc/tutorials/imgproc/histograms/histogram_calculation/histogram_calculation.html

In the code, the color conversion:
Imgproc.cvtColor(image, image, Imgproc.COLOR_RGB2GRAY);
should be to HSV not gray:
Imgproc.cvtColor(image, image, Imgproc.COLOR_BGR2HSV);
In your example you will only have one (gray) plane instead of the 3 HSV channels. That will give errors when you access the 2nd and 3rd plane.

Here is the code for comparing the histogram of Source image to a reference image for OpenCV 2.4.11 Java (Android).
// Assume SourceImage is a Bitmap ARGB_8888
BitmapFactory.Options options = new BitmapFactory.Options();
options.inPreferredConfig = Bitmap.Config.ARGB_8888;
Bitmap refImage = BitmapFactory.decodeFile(mBaseDir + "some_reference.jpg", options);
Mat hsvRef = new Mat();
Mat hsvSource = new Mat();
Mat srcRef = new Mat(refImage.getHeight(), refImage.getWidth(), CvType.CV_8U, new Scalar(4));
Utils.bitmapToMat(refImage, srcRef);
Mat srcSource = new Mat(SourceImage.getHeight(), SourceImage.getWidth(), CvType.CV_8U, new Scalar(4));
Utils.bitmapToMat(SourceImage, srcSource);
/// Convert to HSV
Imgproc.cvtColor(srcRef, hsvRef, Imgproc.COLOR_BGR2HSV);
Imgproc.cvtColor(srcSource, hsvSource, Imgproc.COLOR_BGR2HSV);
/// Using 50 bins for hue and 60 for saturation
int hBins = 50;
int sBins = 60;
MatOfInt histSize = new MatOfInt( hBins, sBins);
// hue varies from 0 to 179, saturation from 0 to 255
MatOfFloat ranges = new MatOfFloat( 0f,180f,0f,256f );
// we compute the histogram from the 0-th and 1-st channels
MatOfInt channels = new MatOfInt(0, 1);
Mat histRef = new Mat();
Mat histSource = new Mat();
ArrayList<Mat> histImages=new ArrayList<Mat>();
histImages.add(hsvRef);
Imgproc.calcHist(histImages,
channels,
new Mat(),
histRef,
histSize,
ranges,
false);
Core.normalize(histRef,
histRef,
0,
1,
Core.NORM_MINMAX,
-1,
new Mat());
histImages=new ArrayList<Mat>();
histImages.add(hsvSource);
Imgproc.calcHist(histImages,
channels,
new Mat(),
histSource,
histSize,
ranges,
false);
Core.normalize(histSource,
histSource,
0,
1,
Core.NORM_MINMAX,
-1,
new Mat());
double resp1 = Imgproc.compareHist(histRef, histSource, 0);
double resp2 = Imgproc.compareHist(histRef, histSource, 1);
double resp3 = Imgproc.compareHist(histRef, histSource, 2);
double resp4 = Imgproc.compareHist(histRef, histSource, 3);

The next code works fine for one depth channel. You have to do just a few modifications to add the other two channels
//Calculate histogram
java.util.List<Mat> matList = new LinkedList<Mat>();
matList.add(imageIR_gray);
Mat histogram = new Mat();
MatOfFloat ranges=new MatOfFloat(0,256);
MatOfInt histSize = new MatOfInt(255);
Imgproc.calcHist(
matList,
new MatOfInt(0),
new Mat(),
histogram ,
histSize ,
ranges);
// Create space for histogram image
Mat histImage = Mat.zeros( 100, (int)histSize.get(0, 0)[0], CvType.CV_8UC1);
// Normalize histogram
Core.normalize(histogram, histogram, 1, histImage.rows() , Core.NORM_MINMAX, -1, new Mat() );
// Draw lines for histogram points
for( int i = 0; i < (int)histSize.get(0, 0)[0]; i++ )
{
Core.line(
histImage,
new org.opencv.core.Point( i, histImage.rows() ),
new org.opencv.core.Point( i, histImage.rows()-Math.round( histogram.get(i,0)[0] )) ,
new Scalar( 255, 255, 255),
1, 8, 0 );
}