I'm new to android and my application crashes after some time when I use drawcontour. The application has to look for the largest contour and compute its bounding box. Here is a sample of my code:
public Mat onCameraFrame(CvCameraViewFrame inputFrame) {
img_rgb = inputFrame.rgba();
Imgproc.blur(img_rgb, blur, new Size(25,25));
Imgproc.cvtColor(blur, img_gray, Imgproc.COLOR_BGR2GRAY);
Imgproc.threshold(img_gray, thres1, 0, 255, Imgproc.THRESH_OTSU);
Imgproc.erode(thres1, erode1, new Mat(), new Point(-1,-1), 3);
Imgproc.dilate(erode1, dilate1, new Mat(), new Point(-1,-1), 2);
Imgproc.erode(dilate1, erode2, new Mat(), new Point(-1,-1), 3);
Imgproc.dilate(erode2, dilate2, new Mat(), new Point(-1,-1), 2);
Imgproc.adaptiveThreshold(dilate2, thres2, 128,Imgproc.ADAPTIVE_THRESH_MEAN_C, Imgproc.THRESH_BINARY_INV,7, 1);
contours = new ArrayList<MatOfPoint>();
Imgproc.findContours(thres2, contours, hierarchy, Imgproc.RETR_TREE, Imgproc.CHAIN_APPROX_SIMPLE, new Point(0, 0));
hierarchy.release();
//Imgproc.drawContours(img_rgb, contours, -1, new Scalar(Math.random()*255, Math.random()*255, Math.random()*255));//, 2, 8, hierarchy, 0, new Point());
for(int i = 0;i<contours.size();i++){
double area = Imgproc.contourArea(contours.get(i));
if(area > largest_area){
largest_area = area;
largest_contour_index = i;
bounding_rect = Imgproc.boundingRect(contours.get(i));
}
}
Imgproc.drawContours(img_rgb,contours,largest_contour_index, new Scalar(0,255,0),2);
return img_rgb;
}
You never reset largest_area and largest_contour_index, so it may happen that largest_contour_index refers to an invalid index.
You only need to reset these values at each loop, and draw the contour only if the index is valid (i.e. if you find at least one contour). Depending on your further processing, you may want also to reset bounding_rect. You can move the boundingRect call out of the for loop, so you avoid useless computation.
Code:
// Reset values at each iteration
largest_area = 0;
largest_contour_index = -1;
for(int i = 0;i<contours.size();i++){
double area = Imgproc.contourArea(contours.get(i));
if(area > largest_area){
largest_area = area;
largest_contour_index = i;
}
}
// Draw only if index is valid
if(largest_contour_index >= 0) {
bounding_rect = Imgproc.boundingRect(contours.get(largest_contour_index));
Imgproc.drawContours(img_rgb,contours,largest_contour_index, new Scalar(0,255,0),2);
}
Related
I am trying to determine a way to change the pixel color of my masks from black to a different color. Unfortunately, I have not be able to determine a way to do this task. Essentially, what I am trying to do is take this image:
and convert the black portions to a color with values (255, 160, 130). I have tried several methods to try and achieve my goal. These include draw contours, setTo, and looping through the matrix. Unfortunately all of these attempts have failed. I have included the code and the resulting outcomes below.
Draw Contours method
System.loadLibrary(Core.NATIVE_LIBRARY_NAME);
Mat img = Imgcodecs.imread(
"C:\\Users\\Hassan\\Documents\\School\\Me\\COMP5900 Y\\Project\\Project\\src\\resources\\face.jpg");
Mat img_grey = new Mat();
Mat grad = new Mat(), grad_x = new Mat(), grad_y = new Mat();
Mat abs_grad_x = new Mat(), abs_grad_y = new Mat();
int ddepth = CvType.CV_32F;
int scale = 1;
int delta = 0;
Imgproc.GaussianBlur(img, img, new Size(3, 3), 0, 0, Core.BORDER_CONSTANT);
Imgproc.cvtColor(img, img_grey, Imgproc.COLOR_BGR2GRAY);
// Apply Sobel
Imgproc.Sobel(img_grey, grad_x, ddepth, 1, 0, 3, scale, delta, Core.BORDER_DEFAULT);
Imgproc.Sobel(img_grey, grad_y, ddepth, 0, 1, 3, scale, delta, Core.BORDER_DEFAULT);
// converting back to CV_8U
Core.convertScaleAbs(grad_x, abs_grad_x);
Core.convertScaleAbs(grad_y, abs_grad_y);
// Total Gradient (approximate)
Core.addWeighted(abs_grad_x, 0.5, abs_grad_y, 0.5, 0, grad);
Photo.fastNlMeansDenoising(grad, grad);
Imgproc.GaussianBlur(grad, grad, new Size(3, 3), 0, 0, Core.BORDER_CONSTANT);
// isolate background
Mat background = new Mat();
Imgproc.threshold(grad, background, 2, 255, Imgproc.THRESH_BINARY);
// draw contours
List<MatOfPoint> contours = new ArrayList<>();
Mat hierarchy = new Mat();
Imgproc.findContours(background, contours, hierarchy, Imgproc.RETR_TREE, Imgproc.CHAIN_APPROX_NONE);
Mat drawing = Mat.zeros(background.size(), CvType.CV_8UC3);
List<MatOfPoint> hullList = new ArrayList<>();
for (MatOfPoint contour : contours) {
MatOfInt hull = new MatOfInt();
Imgproc.convexHull(contour, hull);
Point[] contourArray = contour.toArray();
Point[] hullPoints = new Point[hull.rows()];
List<Integer> hullContourIdxList = hull.toList();
for (int i = 0; i < hullContourIdxList.size(); i++) {
hullPoints[i] = contourArray[hullContourIdxList.get(i)];
}
hullList.add(new MatOfPoint(hullPoints));
}
for (int i = 0; i < contours.size(); i++) {
Scalar color = new Scalar(255, 160, 130);
Imgproc.drawContours(drawing, contours, i, color);
//Imgproc.drawContours(drawing, hullList, i, color );
}
Note here, that I also tried using Imgproc.RETR_EXTERNAL as well, but that produced a completely black image. Also the name of the HighGui window is called "flood fill", but I just forgot to update the name.
setTo
// replace find and draw contours portion of code above
Mat out = new Mat();
background.copyTo(out);
out.setTo(new Scalar(255, 160, 130), background);
Iterating through matrix
// replace draw contours portion of code above
for (a = 0; a < background.rows(); a++) {
for(b = 0; b < background.cols(); b++) {
if(background.get(a,b)[0] == 0) {
//background.put(a, b, CvType.CV_16F, new Scalar(255, 160, 130));
double[] data = {255, 160, 130};
background.put(a, b, data);
}
}
}
The loop is promising, but I know it will not be efficient as I have 2 other masks that I would like to update as well. Could you please suggest an efficient method, that allows me to set the value for all three channels?
Thanks
I am not sure why you are doing many operations on the image but to me it looks like applying the mask and replacing the color efficiently. So if there are other complexities than please let me know.
Below is the code I was looking for in Java.
public static void main(String s[]) {
System.loadLibrary(Core.NATIVE_LIBRARY_NAME);
Mat matr =Imgcodecs.imread("/home/shariq/Desktop/test.png");
Mat result = new Mat();
//create a mask based on range
Core.inRange(matr, new Scalar(0), new Scalar(50), result);
Imgcodecs.imwrite("/home/shariq/Desktop/test_in.png", result);
//apply the mask with color you are looking for, note here scalar is in hsv
matr.setTo(new Scalar(130,160,255),result);
Imgcodecs.imwrite("/home/shariq/Desktop/result.png", matr);
}
We are creating a mask for the pixel values between 0-50 for black color using inRange method.
Core.inRange(matr, new Scalar(0), new Scalar(50), result);
This mask in result variable is than applied to original matrix using setTo method. The replacement color value is provided in HSV format through Scalar object. new Scalar(a,b,c) in HSV can be understand in RGB like this Red = c, Green = b and Blue = a.
matr.setTo(new Scalar(130,160,255),result);
Its quite fast compared to iterating the pixels one by one.
I am developing an Android application and I'm processing frames using Opencv in real time.
The image processing is heavy as I'm detecting the largest contour, rotating and cropping it and apply some sort of segmentation on the cropped Mat object.
The problem is that FPS start with a value of 3 and drops to 1.
I'm not doing some heavy operations like matToBitmap and I'm not making native calls yet.
It's my first big project and I don't have much experience in Opencv. What can I do to increase my FPS?
Here is my code:
long e1 = Core.getTickCount();
Mat mGray = new Mat();
MatOfDouble mu = new MatOfDouble();
MatOfDouble stddev = new MatOfDouble();
Imgproc.cvtColor(origMat, origMat, Imgproc.COLOR_BGRA2BGR);
Imgproc.cvtColor(origMat, mGray, Imgproc.COLOR_BGR2GRAY);
Core.meanStdDev(mGray, mu, stddev);
Imgproc.GaussianBlur(mGray, mGray, new Size(5, 5), 5);
Imgproc.Canny(mGray, mGray, (mu.get(0, 0)[0]) * 0.66, (mu.get(0, 0)[0]) * 1.33, 3, false);
Mat kernell = Imgproc.getStructuringElement(Imgproc.MORPH_RECT, new Size(9, 9));
Imgproc.morphologyEx(mGray, mGray, Imgproc.MORPH_CLOSE, kernell);
Imgproc.dilate(mGray, mGray, Imgproc.getStructuringElement(Imgproc.MORPH_CROSS, new Size(3, 3)));
List<MatOfPoint> contours = new ArrayList<MatOfPoint>();
Mat hierarchy = new Mat();
Imgproc.findContours(mGray, contours, hierarchy, Imgproc.RETR_EXTERNAL, Imgproc.CHAIN_APPROX_SIMPLE);
double largest_area = 0;
Rect rect = new Rect();
int largest_idx = 0;
for (int idx = 0; idx < contours.size(); idx++) {
double a = Imgproc.contourArea(contours.get(idx)); //Find the area of contour
if (a > largest_area) {
largest_area = a;
largest_idx = idx;
}
}
if (contours.size() > 0) {
MatOfPoint2f new_mat = new MatOfPoint2f(contours.get(largest_idx).toArray());
RotatedRect rbox = Imgproc.minAreaRect(new_mat);
Point vertices[] = new Point[4];
rbox.points(vertices);
List<MatOfPoint> boxContours = new ArrayList<>();
boxContours.add(new MatOfPoint(vertices));
for (int i = 0; i < 4; ++i) {
Imgproc.line(origMat, vertices[i], vertices[(i + 1) % 4], new Scalar(255, 0, 0));
}
double rect_angle = rbox.angle - 90.0f;
Size rect_size = rbox.size;
double d = rect_size.width;
rect_size.width = rect_size.height;
rect_size.height = d;
M = Imgproc.getRotationMatrix2D(rbox.center, rect_angle, 1.0);
Imgproc.warpAffine(origMat, rotated, M, origMat.size());
if (rect_size.width > 70 && rect_size.height > 70)
Imgproc.getRectSubPix(rotated, new Size(rect_size.width - 70, rect_size.height - 70), rbox.center, rotated);
Imgproc.resize(rotated, rotated, origMat.size());
Mat orr = rotated.clone();
Imgproc.cvtColor(orr, orr, Imgproc.COLOR_RGB2HSV);
Core.split(orr, channels);
orr = channels.get(2);
CLAHE clahe = Imgproc.createCLAHE();
clahe.setClipLimit(1);
clahe.apply(orr, orr);
Imgproc.GaussianBlur(orr, orr, new Size(5, 5), 5);
Core.meanStdDev(mGray, mu, stddev);
Imgproc.Canny(orr, orr, (mu.get(0, 0)[0]) * 0.66, (mu.get(0, 0)[0]) * 1.33, 3, false);
Imgproc.morphologyEx(orr, orr, Imgproc.MORPH_CLOSE, kernell);
Imgproc.dilate(orr, orr, Imgproc.getStructuringElement(Imgproc.MORPH_CROSS, new Size(3, 3)));
Imgproc.findContours(orr, contours, new Mat(), Imgproc.RETR_EXTERNAL, Imgproc.CHAIN_APPROX_SIMPLE);
for (int i = contours.size() - 1; i >= 0; i--) {
double area = Imgproc.contourArea(contours.get(i), false); // Find the area of contour
if (area < min_area)
contours.remove(i);
}
if (contours.size() > 0) {
MatOfPoint2f approxCurve = new MatOfPoint2f();
rectList.clear();
for (int idx = 0; idx < contours.size(); idx++) {
//Convert contours(i) from MatOfPoint to MatOfPoint2f
MatOfPoint2f contour2f = new MatOfPoint2f(contours.get(idx).toArray());
//Processing on mMOP2f1 which is in type MatOfPoint2f
double approxDistance = Imgproc.arcLength(contour2f, true) * 0.02;
Imgproc.approxPolyDP(contour2f, approxCurve, approxDistance, true);
//Convert back to MatOfPoint
MatOfPoint points = new MatOfPoint(approxCurve.toArray());
// Get bounding rect of contour
Rect rectt = Imgproc.boundingRect(points);
rectList.add(rect);
Mat miniature = new Mat(orr, new Rect(rectt.tl(), rectt.br()));
mats.add(miniature);
// draw enclosing rectangle (all same color, but you could use variable i to make them unique)
Imgproc.rectangle(rotated, rectt.tl(), rectt.br(), new Scalar(255, 0, 0));
}
}
long e2 = Core.getTickCount();
long e = e2 - e1;
double time = e / Core.getTickFrequency();
Log.d("timeTAG", "" + time);
return orr;
}
return rotated;
I am performing Canny edge detector using Android and Opencv on an image to detect the largest contour, extract it using warpPerspective method then find all objects inside that contour. Everything is working as expected but only for image that isn't rotated.
I am using boundingRect to get the contour and use its coordinates to extract it.
Here my code:
private Mat detectLargestContour(Mat origMat) {
// long e1 = Core.getTickCount();
Mat mGray = new Mat();
MatOfDouble mu = new MatOfDouble();
MatOfDouble stddev = new MatOfDouble();
Imgproc.cvtColor(origMat, mGray, Imgproc.COLOR_BGR2GRAY);
Core.meanStdDev(mGray, mu, stddev);
Imgproc.GaussianBlur(mGray, mGray, new Size(5, 5), 5);
//Imgproc.Canny(mGray, mGray, 30, 80, 3, false); //FOR HIGH BRIGHTNESS
//Imgproc.Canny(mGray, mGray, 50, 130, 3, false); // FOR LOW BRIGHTNESS
Imgproc.Canny(mGray, mGray, mu.get(0, 0)[0], stddev.get(0, 0)[0], 3, false);
Mat kernell = Imgproc.getStructuringElement(Imgproc.MORPH_RECT, new Size(9,9));
Imgproc.morphologyEx(mGray, mGray, Imgproc.MORPH_CLOSE, kernell);
Imgproc.dilate(mGray, mGray, Imgproc.getStructuringElement(Imgproc.MORPH_CROSS, new Size(3, 3)));
List<MatOfPoint> contours = new ArrayList<MatOfPoint>();
Mat hierarchy = new Mat();
Imgproc.findContours(mGray, contours, hierarchy, Imgproc.RETR_EXTERNAL, Imgproc.CHAIN_APPROX_SIMPLE);
//MatOfPoint2f approxCurve = new MatOfPoint2f();
double largest_area=0;
Rect rect = new Rect();
for (int idx = 0; idx < contours.size() ; idx++) {
double a = Imgproc.contourArea(contours.get(idx)); //Find the area of contour
if (a > largest_area) {
largest_area = a;
rect = Imgproc.boundingRect(contours.get(idx));
}
}
if (rect.area() > 100000) {
Imgproc.rectangle(origMat, rect.tl(), rect.br(), new Scalar(0, 255, 0));
p1 = new Point(rect.tl().x, rect.tl().y);
p2 = new Point(rect.tl().x + rect.width, rect.tl().y);
p3 = new Point(rect.tl().x, rect.tl().y + rect.height);
p4 = new Point(rect.tl().x + rect.width, rect.tl().y + rect.height);
card_corners = new ArrayList<>();
card_corners.add(p1);
card_corners.add(p3);
card_corners.add(p4);
card_corners.add(p2);
warpedCard = new Mat(origMat.rows(), origMat.cols(), CvType.CV_8UC3);
final Point p1 = new Point(warpedCard.cols() + marge, warpedCard.rows() + marge);
final Point p2 = new Point(0 - marge, warpedCard.rows() + marge);
final Point p3 = new Point(0 - marge, 0 - marge);
final Point p4 = new Point(warpedCard.cols() + marge, 0 - marge);
LinkedList<Point> sceneList = new LinkedList<Point>();
sceneList.addLast(p4);
sceneList.addLast(p3);
sceneList.addLast(p2);
sceneList.addLast(p1);
MatOfPoint2f scene = new MatOfPoint2f();
scene.fromList(sceneList);
MatOfPoint2f obj = new MatOfPoint2f();
obj.fromList(card_corners);
Mat homography = Calib3d.findHomography(obj, scene);
Imgproc.warpPerspective(origMat, warpedCard, homography, new Size(warpedCard.cols(), warpedCard.rows()));
return warpedCard;
}
return origMat;
}
It's weird but only boundingRect gave me a stable and performant result but the drawn rectangle doesn't rotate with the found contour.
How can I manage to resolve this issue? Any thoughts?
EDIT:
I changed boundingRect with minAreaRect.
Here is the code
int largest_idx = 0;
for (int idx = 0; idx < contours.size() ; idx++) {
double a = Imgproc.contourArea(contours.get(idx)); //Find the area of contour
if (a > largest_area) {
largest_area = a;
// rect = Imgproc.boundingRect(contours.get(idx));
largest_idx = idx;
}
}
MatOfPoint2f new_mat = new MatOfPoint2f( contours.get(largest_idx).toArray() );
RotatedRect rbox = Imgproc.minAreaRect(new_mat);
Log.d("rotatedrect_angle", "" + rbox.angle);
Point points[] = new Point[4];
rbox.points(points);
for(int i=0; i<4; ++i){
Imgproc.line(origMat, points[i], points[(i+1)%4], new Scalar(255,255,255));
}
And here is what I've got:
As you can see the detection isn't as accurate as when I used boundingRect.
A Python demo to find and draw a rotatedRect:
# 2019/03/01
# https://stackoverflow.com/a/54942835/3547485
import numpy as np
import cv2
gray = cv2.imread("tmp.png", cv2.IMREAD_GRAYSCALE)
th, threshed = cv2.threshold(gray, 220, 255, cv2.THRESH_BINARY_INV)
cnts = cv2.findContours(threshed, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)[-2]
cnt = sorted(cnts, key=cv2.contourArea, reverse=True)[0]
rbox = cv2.minAreaRect(cnt)
pts = cv2.boxPoints(rbox).astype(np.int32)
cv2.drawContours(img, [pts], -1, (0, 255, 0), 1, cv2.LINE_AA)
cv2.imwrite("dst.png", img)
Useful OpenCV functions(in Python) : cv2.minAreaRect, cv2.boxPoints, cv.2drawContours. You can find corresponding functions in Java.
I am working on a licence plate recognition software using OpenCV, Tesseract and Java but experiencing issues, I cant seem to segment my text correctly, its not always that I get all characters to be detected and bounded with a bounding box these are some of my outputs with my code...and also when I detect the characters I never know which character is in which box so when I pass them through tesseract they get jumbled, how do I format my string?
This one fails despite the clearly visible characters:
Here Z and 6 fail to be detected even when 6 is clearly visible:
Below is my code:
originalFrame = image.clone();
roiColor = image.clone();
Imgproc.cvtColor(image, image, Imgproc.COLOR_BGR2GRAY, 0);
originalFrameGrayScale = image.clone();
Mat morph = Imgproc.getStructuringElement(Imgproc.MORPH_RECT, new Size(9, 9));
Imgproc.morphologyEx(image, image, Imgproc.MORPH_TOPHAT, morph);
Imgproc.Sobel(image, image, -1, 2, 0);
Imgproc.GaussianBlur(image, image, new Size(5,5), 3,3);
Imgproc.morphologyEx(image, image, Imgproc.MORPH_CLOSE, morph);
Imgproc.threshold(image, image, 200, 255, Imgproc.THRESH_OTSU);
Vector<Rect> rectangles = detectionContour(image);
Mat roi = originalFrameGrayScale.clone();
if(!rectangles.isEmpty()){
roi = originalFrameGrayScale.submat(rectangles.get(0));
roiBlack = roi.clone();
roiColor = roiColor.submat(rectangles.get(0));
Imgproc.rectangle(originalFrame, rectangles.get(0).br(), rectangles.get(0).tl(), new Scalar(0,0,255), 2);
}
Imgproc.medianBlur(roi, roi, 3);
Imgproc.adaptiveThreshold(roi, roi, 225, Imgproc.ADAPTIVE_THRESH_GAUSSIAN_C, Imgproc.THRESH_BINARY, 15, 3);
roiBinarize = roi.clone();
Mat erode = Imgproc.getStructuringElement(Imgproc.MORPH_RECT, new Size(1, 1));
Mat dilate = Imgproc.getStructuringElement(Imgproc.MORPH_RECT,new Size(1, 1));
Imgproc.morphologyEx(roi, roi, Imgproc.MORPH_OPEN, dilate);
Imgproc.morphologyEx(roi, roi, Imgproc.MORPH_OPEN, erode);
Imgproc.Canny(roi, roi, 150, 150 * 3, 3, true);
Vector<Rect> letters = detectionPlateCharacterContour(roi);
doTesseractOCR(letters, roiBinarize);
private static void doTesseractOCR(Vector<Rect> letters, Mat plate){
Tesseract instance = new Tesseract(); //
instance.setLanguage(LANGUAGE);
String resultPlate = "";
for(int i= 0; i < letters.size(); i++){
BufferedImage letter = OpenCvUtils.Mat2bufferedImage(plate.submat(letters.get(i)));
try {
String result = instance.doOCR(letter);
resultPlate += result + " position "+i;
} catch (TesseractException e) {
System.err.println(e.getMessage());
}
System.out.println("Tesseract output: "+resultPlate);
}
}
private static Vector<Rect> detectionPlateCharacterContour(Mat roi) {
Mat contHierarchy = new Mat();
Mat imageMat = roi.clone();
Rect rect = null;
List<MatOfPoint> contours = new ArrayList<>();
Imgproc.findContours(imageMat, contours, contHierarchy, Imgproc.RETR_EXTERNAL, Imgproc.CHAIN_APPROX_NONE);
Vector<Rect> rect_array = new Vector<>();
for (int i = 0; i < contours.size(); i++) {
rect = Imgproc.boundingRect(contours.get(i));
double ratio = 0;
if(rect.height > rect.width){
ratio = rect.height/rect.width;
}else{
ratio = rect.width/rect.height;
}
Logger.printMessage("Ratio of letter: "+ratio);
double contourarea = Imgproc.contourArea(contours.get(i));
if (contourarea >= 100 && contourarea <= 1000 && ( ratio >= 1 && ratio <= 2)) {
Imgproc.rectangle(roiColor, rect.br(), rect.tl(), new Scalar(255,0,0));
rect_array.add(rect);
}
}
contHierarchy.release();
return rect_array;
}
I try to dilate three image of characters in java opencv. i found out than ever it same character with same font and size, after dilate the result is different. so i try with same image the result is still different. Here is my test code.
for (int j = 0; j < 3; j++) {
Mat InputSrc = openFile("src\\myOpencv\\ocr\\crop1.png");
Mat tempImg =new Mat();
Imgproc.cvtColor(InputSrc, tempImg, Imgproc.COLOR_BGR2GRAY);
Imgproc.threshold(tempImg, tempImg, 0, 255, Imgproc.THRESH_OTSU);
imageViewer.show(tempImg, "src");
Mat kernal5 = new Mat(5, 5, CV_8U);
Point midPoint = new Point(-1, -1);
Scalar scalarOne = new Scalar(1);
Mat binImg2 = new Mat();
Imgproc.dilate(tempImg, binImg2, kernal5, midPoint, 1, 1, scalarOne);
imageViewer.show(binImg2, "dilate");
}
thank