We Keep Coding

How to adjust the threshold for template matching in openCV (java)?

I am running template matching using openCV 3.4.7 Android SDK (java).
The code work almost perfectly; when the template is match, it draws a rectangle on the matching area. The problem is that even when there is no match, it draws a random rectangle. I think that happens because the threshold is not set correctly. If so, can someone please help me out?
Here's the code:
public static void run(String inFile, String templateFile, String outFile,
int match_method) {
Mat img = Imgcodecs.imread(inFile);
Mat templ = Imgcodecs.imread(templateFile);
// / Create the result matrix
int result_cols = img.cols() - templ.cols() + 1;
int result_rows = img.rows() - templ.rows() + 1;
Mat result = new Mat(result_rows, result_cols, CvType.CV_32FC1);
// / Do the Matching and Normalize
Imgproc.matchTemplate(img, templ, result, match_method);
Core.normalize(result, result, 0, 1, Core.NORM_MINMAX, -1, new Mat());
// / Localizing the best match with minMaxLoc
Core.MinMaxLocResult mmr = Core.minMaxLoc(result);
Point matchLoc;
if (match_method == Imgproc.TM_SQDIFF
|| match_method == Imgproc.TM_SQDIFF_NORMED) {
matchLoc = mmr.minLoc;
} else {
matchLoc = mmr.maxLoc;
}
// / Show me what you got
Imgproc.rectangle(img, matchLoc, new Point(matchLoc.x + templ.cols(),
matchLoc.y + templ.rows()), new Scalar(0, 0, 128));
// Save the visualized detection.
System.out.println("Writing " + outFile);
Imgcodecs.imwrite(outFile, img);
}

You can use Imgproc.TM_CCOEFF_NORMED or Imgproc.TM_CCORR_NORMED and mmr.maxVal >= 0.8. It should take care of most of your false positives.
Sample Code:
import org.opencv.core.*;
import org.opencv.imgcodecs.Imgcodecs;
import org.opencv.imgproc.Imgproc;
import java.io.File;
import java.nio.file.Files;
public class templateMatchingTester {
private static String str = null;
static {
if (str == null) {
str = "initialised";
nu.pattern.OpenCV.loadShared();
System.loadLibrary(Core.NATIVE_LIBRARY_NAME);
}
}
private static Mat createMatrixFromImage(String imagePath) {
Mat imageMatrix = Imgcodecs.imread(imagePath);
Mat greyImage = new Mat();
Imgproc.cvtColor(imageMatrix, greyImage, Imgproc.COLOR_BGR2GRAY);
return greyImage;
}
private static boolean matchTemplate(String pathToInputImage,String pathToTemplate){
Mat inputImage = createMatrixFromImage(pathToInputImage);
Mat templateImage = createMatrixFromImage(pathToTemplate);
// Create the result matrix
int result_cols = inputImage.cols() - templateImage.cols() + 1;
int result_rows = inputImage.rows() - templateImage.rows() + 1;
Mat result = new Mat(result_rows, result_cols, CvType.CV_8UC1);
int match_method;
match_method = Imgproc.TM_CCOEFF_NORMED;//Imgproc.TM_CCORR_NORMED;
Imgproc.matchTemplate(inputImage, templateImage, result, match_method);
Core.MinMaxLocResult mmr = Core.minMaxLoc(result);
double minMatchQuality = 0.85;
System.out.println(mmr.maxVal);
if (mmr.maxVal >= minMatchQuality){
return true;
} else
return false;
}
public static void main(String args[]) {
String template = "path/to/your/templateImage";
final File folder = new File("path/to/your/testImagesFolder/");
int matchCount = 0;
for (final File fileEntry : folder.listFiles()){
if (matchTemplate(fileEntry.getPath(),template)){
matchCount+=1;
}else
System.out.println(fileEntry.getPath());
}
System.out.println(matchCount);
}
}

Use a normed match method to ensure your match value is [0..1].
Replace this line
Core.normalize(result, result, 0, 1, Core.NORM_MINMAX, -1, new Mat());
with a thresholding operation. Otherwise a best match of 0.9 would become 1 by the second normalization and you would lose the actual match "quality" information.
Normalizing the result of the template matching will always result in your best match being 1 making it impossible to discard a bad match.

i wrote an app that would take a screenshot of the game overwatch and attempt to tell who is on each team. using template matching and open cv. project need to iterate over the result image and check values.
OpenCVUtils.getPointsFromMatAboveThreshold(result,
0.90f)
public static void scaleAndCheckAll(String guid){
Mat source = imread(IMG_PROC_PATH + guid); //load the source image
Mat scaledSrc = new Mat(defaultScreenshotSize, source.type());
resize(source, scaledSrc, defaultScreenshotSize);
Mat sourceGrey = new Mat(scaledSrc.size(), CV_8UC1);
cvtColor(scaledSrc, sourceGrey, COLOR_BGR2GRAY);
for (String hero : getCharacters()) {
Mat template = OpenCVUtils.matFromJar(TEMPLATES_FOLDER + hero + ".png", 0); //load a template
Size size = new Size(sourceGrey.cols()-template.cols()+1, sourceGrey.rows()-template.rows()+1);
Mat result = new Mat(size, CV_32FC1);
matchTemplate(sourceGrey, template, result, TM_CCORR_NORMED);// get results
Scalar color = OpenCVUtils.randColor();
List<Point> points = OpenCVUtils.getPointsFromMatAboveThreshold(result,
0.90f);
for (Point point : points) {
//rectangle(scaledSrc, new Rect(point.x(),point.y(),template.cols(),template.rows()), color, -2, 0, 0);
putText(scaledSrc, hero, point, FONT_HERSHEY_PLAIN, 2, color);
}
}
String withExt = IMG_PROC_PATH + guid +".png";
imwrite(withExt, scaledSrc);
File noExt = new File(IMG_PROC_PATH + guid);
File ext = new File(withExt);
noExt.delete();
ext.renameTo(noExt);
}
the other method.
public static List<Point> getPointsFromMatAboveThreshold(Mat m, float t){
List<Point> matches = new ArrayList<Point>();
FloatIndexer indexer = m.createIndexer();
for (int y = 0; y < m.rows(); y++) {
for (int x = 0; x < m.cols(); x++) {
if (indexer.get(y,x)>t) {
System.out.println("(" + x + "," + y +") = "+ indexer.get(y,x));
matches.add(new Point(x, y));
}
}
}
return matches;
}
you can just get the first from the list or see how close they are if you expect multiple matches.

Compare Camera captured images with computer generated images

I am making an app which compare camera captured image with computer generated images and these images are stored in database.
I am using Opencv for image comparison but it only compare computer generated images correctly.When i capture same image with camera and compare that image with computer generated image then it does not match them acuurately.Is there any way where i can compare camera images also;
Here is my code
public static void main(String[] args) {
// Set image path
String path = System.getProperty("user.dir");
String filename1 = path+"/images/266.jpg";
String filename2 = path+"/images/20160805_121529.jpg";
int ret;
ret = compareFeature(filename1, filename2);
if (ret > 0) {
System.out.println("Two images are same.");
} else {
System.out.println("Two images are different.");
}
}
/**
* Compare that two images is similar using feature mapping
* #author minikim
* #param filename1 - the first image
* #param filename2 - the second image
* #return integer - count that has the similarity within images
*/
public static int compareFeature(String filename1, String filename2) {
int retVal = 0;
long startTime = System.currentTimeMillis();
System.loadLibrary(Core.NATIVE_LIBRARY_NAME);
// Load images to compare
Mat img1 = Imgcodecs.imread(filename1, Imgcodecs.CV_LOAD_IMAGE_COLOR);
Mat img2 = Imgcodecs.imread(filename2, Imgcodecs.CV_LOAD_IMAGE_COLOR);
// Declare key point of images
MatOfKeyPoint keypoints1 = new MatOfKeyPoint();
MatOfKeyPoint keypoints2 = new MatOfKeyPoint();
Mat descriptors1 = new Mat();
Mat descriptors2 = new Mat();
// Definition of ORB key point detector and descriptor extractors
FeatureDetector detector = FeatureDetector.create(FeatureDetector.ORB);
DescriptorExtractor extractor = DescriptorExtractor.create(DescriptorExtractor.ORB);
// Detect key points
detector.detect(img1, keypoints1);
detector.detect(img2, keypoints2);
// Extract descriptors
extractor.compute(img1, keypoints1, descriptors1);
extractor.compute(img2, keypoints2, descriptors2);
// Definition of descriptor matcher
DescriptorMatcher matcher = DescriptorMatcher.create(DescriptorMatcher.BRUTEFORCE_HAMMING);
// Match points of two images
MatOfDMatch matches = new MatOfDMatch();
// System.out.println("Type of Image1= " + descriptors1.type() + ", Type of Image2= " + descriptors2.type());
// System.out.println("Cols of Image1= " + descriptors1.cols() + ", Cols of Image2= " + descriptors2.cols());
// Avoid to assertion failed
// Assertion failed (type == src2.type() && src1.cols == src2.cols && (type == CV_32F || type == CV_8U)
if (descriptors2.cols() == descriptors1.cols()) {
matcher.match(descriptors1, descriptors2 ,matches);
// Check matches of key points
DMatch[] match = matches.toArray();
double max_dist = 0; double min_dist = 100;
for (int i = 0; i < descriptors1.rows(); i++) {
double dist = match[i].distance;
if( dist < min_dist ) min_dist = dist;
if( dist > max_dist ) max_dist = dist;
}
System.out.println("max_dist=" + max_dist + ", min_dist=" + min_dist);
// Extract good images (distances are under 10)
for (int i = 0; i < descriptors1.rows(); i++) {
if (match[i].distance <= 10) {
retVal++;
}
}
System.out.println("matching count=" + retVal);
}
long estimatedTime = System.currentTimeMillis() - startTime;
System.out.println("estimatedTime=" + estimatedTime + "ms");
return retVal;
}
Here is pictures that i am comparing
And here is camera captured image
Thanks in Advance

Save Mat (openCV) to SharedPreferences Android

I'm writting app, which using KNearest. I wrote code to train model, but every restart app, I must train data again, so I would like to save train data to SharedPreferences once and using it after.
I know that I must convert Mat to byte[] and then to String, but decode is not working, I got error:
(layout == ROW_SAMPLE && responses.rows == nsamples)
|| (layout == COL_SAMPLE && responses.cols == nsamples)
in function void cv::ml::TrainDataImpl::setData(cv::InputArray,
int, cv::InputArray, cv::InputArray,
cv::InputArray, cv::InputArray, cv::InputArray, cv::InputArray)
Code:
protected Void doInBackground(Void... args) {
// Constants.TRAIN_SAMPLES = 10
Mat trainData = new Mat(0, 200 * 200, CvType.CV_32FC1); // 0 x 40 000
Mat trainClasses = new Mat(Constants.TRAIN_SAMPLES, 1, CvType.CV_32FC1); // 10 x 1
float[] myint = new float[Constants.TRAIN_SAMPLES + 1];
for (i = 1; i <= Constants.TRAIN_SAMPLES; i++)
myint[i] = (float) i;
trainClasses.put(0, 0, myint);
KNearest knn = KNearest.create();
String val = " ";
val = sharedPref.getString("key", " ");
// empty SharedPreferences
if (val.equals(" ")) {
// get all images from external storage
for (i = 1; i <= Constants.TRAIN_SAMPLES; i++) {
String photoPath = Environment.getExternalStorageDirectory().toString() + "/ramki/ramka_" + i + ".png";
BitmapFactory.Options options = new BitmapFactory.Options();
options.inPreferredConfig = Bitmap.Config.ARGB_8888;
Bitmap bitmap = BitmapFactory.decodeFile(photoPath, options);
Utils.bitmapToMat(bitmap, img);
if (img.channels() == 3) {
Imgproc.cvtColor(img, img, Imgproc.COLOR_RGB2GRAY);
} else if (img.channels() == 4) {
Imgproc.cvtColor(img, img, Imgproc.COLOR_RGBA2GRAY);
}
Imgproc.resize(img, img, new Size(200, 200));
img.convertTo(img, CvType.CV_32FC1);
img = img.reshape(1, 1); // 1 x 40 000 ( 200x200 )
trainData.push_back(img);
publishProgress(i);
}
trainData.convertTo(trainData, CvType.CV_8U);
// save this trainData (Mat) to SharedPreferences
saveMatToPref(trainData);
} else {
// get trainData from SharedPreferences
val = sharedPref.getString("key", " ");
byte[] data = Base64.decode(val, Base64.DEFAULT);
trainData.convertTo(trainData, CvType.CV_8U);
trainData.put(0, 0, data);
}
trainData.convertTo(trainData, CvType.CV_32FC1);
knn.train(trainData, Ml.ROW_SAMPLE, trainClasses);
trainClasses.release();
trainData.release();
img.release();
onPostExecute();
return null;
}
public void saveMatToPref(Mat mat) {
if (mat.isContinuous()) {
int cols = mat.cols();
int rows = mat.rows();
byte[] data = new byte[cols * rows];
// there, data contains {0,0,0,0,0,0 ..... } 400 000 items
mat.get(0, 0, data);
String dataString = new String(Base64.encode(data, Base64.DEFAULT));
SharedPreferences.Editor mEdit1 = sharedPref.edit();
mEdit1.putString("key", dataString);
mEdit1.commit();
} else {
Log.i(TAG, "Mat not continuous.");
}
}
When I decode, my trainData look like this:
Mat [ 0*40000*CV_32FC1 ..]
but should: Mat [ 10*40000*CV_32FC1 ..]
Can anybody help me to encode and decode Mat? Thx for help.

As #Miki mention, problem was in types. Now it works, but only with Mat size around 200 x 40 000 in my case, if it's bigger, I have outOfMemory excepion...
String val = " ";
val = sharedPref.getString("key", " ");
// empty SharedPreferences
if ( ! val.equals(" ")) {
// get all images from external storage
for (i = 1; i <= Constants.TRAIN_SAMPLES; i++) {
String photoPath = Environment.getExternalStorageDirectory().toString() + "/ramki/ramka_" + i + ".png";
BitmapFactory.Options options = new BitmapFactory.Options();
options.inPreferredConfig = Bitmap.Config.ARGB_8888;
Bitmap bitmap = BitmapFactory.decodeFile(photoPath, options);
Utils.bitmapToMat(bitmap, img);
if (img.channels() == 3) {
Imgproc.cvtColor(img, img, Imgproc.COLOR_RGB2GRAY);
} else if (img.channels() == 4) {
Imgproc.cvtColor(img, img, Imgproc.COLOR_RGBA2GRAY);
}
Imgproc.resize(img, img, new Size(200, 200));
img.convertTo(img, CvType.CV_32FC1);
img = img.reshape(1, 1); // 1 x 40 000 ( 200x200 )
trainData.push_back(img);
publishProgress(i);
}
// save this trainData (Mat) to SharedPreferences
saveMatToPref(trainData);
} else {
// get trainData from SharedPreferences
val = sharedPref.getString("key", " ");
byte[] data = Base64.decode(val, Base64.DEFAULT);
trainData = new Mat(Constants.TRAIN_SAMPLES, 200 * 200, CvType.CV_32FC1);
float[] f = toFloatArray(data);
trainData.put(0, 0, f);
}
knn.train(trainData, Ml.ROW_SAMPLE, trainClasses);
public void saveMatToPref(Mat mat) {
if (mat.isContinuous()) {
int size = (int)( mat.total() * mat.channels() );
float[] data = new float[ size ];
byte[] b = new byte[ size ];
mat.get(0, 0, data);
b = FloatArray2ByteArray(data);
String dataString = new String(Base64.encode(b, Base64.DEFAULT));
SharedPreferences.Editor mEdit1 = sharedPref.edit();
mEdit1.putString("key", dataString);
mEdit1.commit();
} else {
Log.i(TAG, "Mat not continuous.");
}
}
private static float[] toFloatArray(byte[] bytes) {
ByteBuffer buffer = ByteBuffer.wrap(bytes);
FloatBuffer fb = buffer.asFloatBuffer();
float[] floatArray = new float[fb.limit()];
fb.get(floatArray);
return floatArray;
}
public static byte[] FloatArray2ByteArray(float[] values){
ByteBuffer buffer = ByteBuffer.allocate(4 * values.length);
for (float value : values)
buffer.putFloat(value);
return buffer.array();
}
If someone have better solution, please add.

Mouth detection using openCv returns a lot of circles in only one face

I'm using OpenCv with android studio to detect faces in an image along with the eyes and the mouth in each face. But, the problem is whenever I try to detect the mouth it returns multiple circles in a face which is wrong.
Here is the code I added for mouth detection:
private BaseLoaderCallback mLoaderCallback = new BaseLoaderCallback(this) {
#Override
public void onManagerConnected(int status) {
switch (status) {
...
InputStream iserM = getResources().openRawResource(
R.raw.haarcascade_mcs_mouth);
File cascadeDirERM = getDir("cascadeERM",
Context.MODE_PRIVATE);
File cascadeFileERM = new File(cascadeDirERM,
"haarcascade_mcs_mouth.xml");
FileOutputStream oserM = new FileOutputStream(cascadeFileERM);
byte[] bufferERM = new byte[4096];
int bytesReadERM;
while ((bytesReadERM = iserM.read(bufferERM)) != -1) {
oserM.write(bufferERM, 0, bytesReadERM);
}
iserM.close();
oserM.close();
...
//here begins
mJavaDetectorMouth = new CascadeClassifier(
cascadeFileERM.getAbsolutePath());
if (mJavaDetectorMouth.empty()) {
Log.e(TAG, "Failed to load cascade classifier");
mJavaDetectorMouth = null;
} else
Log.i(TAG, "Loaded cascade classifier from "
+ mCascadeFile.getAbsolutePath());
//here ends
...
}
public Mat onCameraFrame(CameraBridgeViewBase.CvCameraViewFrame inputFrame) {
...
Rect r = facesArray[i];
MatOfRect mouths = new MatOfRect();
Mat faceROI = mRgba.submat(facesArray[i]);
mJavaDetectorMouth.detectMultiScale(faceROI, mouths,1.1,1,1, new org.opencv.core.Size(30, 30), new org.opencv.core.Size());
Rect[] mouthArray = mouths.toArray();
for (int j = 0; j < mouthArray.length; j++){
Point center1 = new Point(facesArray[i].x + mouthArray[j].x + mouthArray[j].width * 0.5,
facesArray[i].y + mouthArray[j].y + mouthArray[j].height * 0.5);
int radius = (int) Math.round(mouthArray[j].width / 2);
Imgproc.circle(mRgba, center1, radius, new Scalar(255, 0, 0), 4, 8, 0);
}
...
}

I've been looking around since I posted the question and tried a lot of things. But finally I found the solution:
I changed:
mJavaDetectorMouth.detectMultiScale(faceROI, mouths,1.1,1,1, new org.opencv.core.Size(30, 30), new org.opencv.core.Size());
To:
mJavaDetectorMouth.detectMultiScale(faceROI, mouths,1.1, 2,
Objdetect.CASCADE_FIND_BIGGEST_OBJECT
| Objdetect.CASCADE_SCALE_IMAGE, new org.opencv.core.Size(30, 30), new org.opencv.core.Size());
and I solved the issue.

opencv - matchTemplate

I am new to opencv.
I have a piece of code that finds a match to my template.
public static void findTemplete(String inFile, String templateFile, String outFile, int match_method) {
System.out.println("\nRunning Template Matching");
Mat img = Highgui.imread(inFile);
Mat templ = Highgui.imread(templateFile);
int result_cols = img.cols() - templ.cols() + 1;
int result_rows = img.rows() - templ.rows() + 1;
Mat result = new Mat(result_rows, result_cols, CvType.CV_32FC1);
Imgproc.matchTemplate(img, templ, result, match_method);
Core.normalize(result, result, 0, 1, Core.NORM_MINMAX, -1, new Mat());
MinMaxLocResult mmr = Core.minMaxLoc(result);
Point matchLoc;
if (match_method == Imgproc.TM_SQDIFF || match_method == Imgproc.TM_SQDIFF_NORMED) {
matchLoc = mmr.minLoc;
} else {
matchLoc = mmr.maxLoc;
}
Core.rectangle(img, matchLoc, new Point(matchLoc.x + templ.cols(),
matchLoc.y + templ.rows()), new Scalar(0, 255, 0));
System.out.println("Writing "+ outFile);
Highgui.imwrite(outFile, img);
}
My problem is that matchTemplate finds "The best match". So if my templete doesnt exists at all in the picture it finds something anyhow.
So how can i set the "Match strength", so that only strong matches will be found.

Remove the Normalize after your match Template. This will not allow the minmaxloc to give you proper numbers.
Here is an example:
double minVal; double maxVal=0; Point minLoc; Point maxLoc;
Point matchLoc;
matchTemplate ( frame, objectToFind, result, CV_TM_CCORR_NORMED );
minMaxLoc ( result, &minVal, &maxVal, &minLoc, &maxLoc, Mat ( ) );
matchLoc = maxLoc;
if ( maxVal > .995 )
{
//we have a good match so do something
}
the .995 is what you change to a different number to help remove the bad matches.
Since we arbitrarily set this number, have it output the current value in your console using something like:
printf("My current maxVal: %f \n", maxVal);
This will help you gauge what you should set your value to.
Hope this helps :)