-
Notifications
You must be signed in to change notification settings - Fork 4
/
main.cpp
263 lines (226 loc) · 7.56 KB
/
main.cpp
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
// OpenCV
#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/video/background_segm.hpp>
#include <opencv2/imgproc/imgproc.hpp>
// C
#include <stdio.h>
// C++
#include <iostream>
#include <sstream>
#include <cmath>
using namespace cv;
using namespace std;
/// Maximum difference threshold to accept an input
#define DIFF_THRESH 230
enum {
MAX_WORDS = 26, // Number of letters
SAMPLE_RATE = 1, // Frame sampling rate (every x loops)
NUM_LAST_LETTERS = 3, // Number of letters to store
MIN_FREQ = 2, // Minimum frequency of last letters
RESET_THRESH = 25000000, // Minimum sum pixel data to automatically reset
THRESH = 200, // Threshold to eliminate background noise
KEY_ESC = 27, // 'esc' key code
};
// Global variables
Ptr<BackgroundSubtractor> pMOG2; // MOG2 Background subtractor
vector<Point> letters[MAX_WORDS];
// Function declarations
void processVideo();
void doSystemCalls(char c);
int main(int argc, char* argv[]) {
// Create GUI windows
namedWindow("Crop Frame");
namedWindow("Foreground");
namedWindow("Contour");
namedWindow("Letter");
// Create Background Subtractor objects (MOG2 approach)
pMOG2 = createBackgroundSubtractorMOG2(2000);
// Preload letter images
for (int i = 0; i < MAX_WORDS; i++) {
char buf[13 * sizeof(char)];
sprintf(buf, "images/%c.png", (char)('a' + i));
Mat im = imread(buf, 1);
if (im.data) {
Mat bwim;
cvtColor(im, bwim, CV_RGB2GRAY);
Mat threshold_output;
vector<Vec4i> hierarchy;
vector<vector<Point> > contours;
// Detect edges using Threshold
threshold( bwim, threshold_output, THRESH, 255, THRESH_BINARY );
findContours(threshold_output, contours, hierarchy, CV_RETR_TREE, CV_CHAIN_APPROX_SIMPLE, Point(0, 0) );
letters[i] = contours[0];
}
}
processVideo();
// Destroy GUI windows
destroyAllWindows();
return EXIT_SUCCESS;
}
// Source: http://stackoverflow.com/questions/21482534/how-to-use-shape-distance-and-common-interfaces-to-find-hausdorff-distance-in-op
int distance_2(vector<Point> a, vector<Point> b) {
int maxDistAB = 0;
for (size_t i = 0; i < a.size(); i++) {
int minB = 1000000;
for (size_t j = 0; j < b.size(); j++) {
int dx = (a[i].x - b[j].x);
int dy = (a[i].y - b[j].y);
int tmpDist = dx*dx + dy*dy;
if (tmpDist < minB) {
minB = tmpDist;
}
if (tmpDist == 0) {
break; // can't get better than equal.
}
}
maxDistAB += minB;
}
return maxDistAB;
}
double distance_hausdorff(vector<Point> a, vector<Point> b) {
int maxDistAB = distance_2(a, b);
int maxDistBA = distance_2(b, a);
int maxDist = max(maxDistAB,maxDistBA);
return sqrt((double)maxDist);
}
void processVideo() {
// Create the capture object
VideoCapture capture = VideoCapture(0);
if (!capture.isOpened()) {
// Error in opening the video input
cerr << "Cannot Open Webcam... " << endl;
exit(EXIT_FAILURE);
}
Mat frame; // current frame
Mat fgMaskMOG2; // fg mask fg mask generated by MOG2 method
int keyboard = 0; // last key pressed
int frames = 0; // number of frames since last sample
int letterCount = 0; // number of letters captured since last display
char lastLetters[NUM_LAST_LETTERS] = {0};
char lastExecLetter = 0; // last letter sent to doSystemCalls()
Mat letterText = Mat::zeros(200, 200, CV_8UC3);
// Read input data
while ((char)keyboard != KEY_ESC) {
// Read the current frame
if (!capture.read(frame)) {
cerr << "Unable to read next frame." << endl;
cerr << "Exiting..." << endl;
exit(EXIT_FAILURE);
}
// Crop Frame to smaller region
cv::Rect myROI(50, 150, 200, 200);
Mat cropFrame = frame(myROI);
// Update the background model
pMOG2->apply(cropFrame, fgMaskMOG2, 0.001);
// Generate Convex Hull
Mat threshold_output;
vector<Vec4i> hierarchy;
vector<vector<Point> > contours;
// Detect edges using Threshold
threshold( fgMaskMOG2, threshold_output, THRESH, 255, THRESH_BINARY );
// Find contours
findContours( threshold_output, contours, hierarchy, CV_RETR_TREE, CV_CHAIN_APPROX_SIMPLE, Point(0, 0) );
// Find largest contour
Mat drawing = Mat::zeros(cropFrame.size(), CV_8UC3);
double largest_area = 0;
int maxIndex = 0;
for (int j = 0; j < contours.size(); j++) {
double area = contourArea(contours[j], false); // Find the area of contour
if (area > largest_area) {
largest_area = area;
maxIndex = j; // Store the index of largest contour
}
}
// Draw Largest Contours
Scalar color = Scalar(0, 0, 255);
drawContours(drawing, contours, maxIndex, Scalar(255, 255, 255), CV_FILLED); // fill white
// Draw Contours
Mat contourImg = Mat::zeros(cropFrame.size(), CV_8UC3);
drawContours( contourImg, contours, maxIndex, Scalar(0, 0, 255), 2, 8, hierarchy, 0, Point(0, 0) );
// Reset if too much noise
Scalar sums = sum(drawing);
int s = sums[0] + sums[1] + sums[2] + sums[3];
if (s >= RESET_THRESH) {
pMOG2 = createBackgroundSubtractorMOG2(2000);
continue;
}
// Compare to reference images
if (contours.size() > 0 && frames++ > SAMPLE_RATE && contours[maxIndex].size() >= 5) {
RotatedRect testRect = fitEllipse(contours[maxIndex]);
frames = 0;
double lowestDiff = HUGE_VAL;
char best = 0;
for (int i = 0; i < MAX_WORDS; i++) {
if (letters[i].size() == 0) continue;
// Match Shapes functions (possible alternative)
/* double diff = matchShapes(letters[i], contours[maxIndex],
CV_CONTOURS_MATCH_I3, 0);
diff += matchShapes(letters[i], contours[maxIndex],
CV_CONTOURS_MATCH_I2, 0);
diff += matchShapes(letters[i], contours[maxIndex],
CV_CONTOURS_MATCH_I1, 0); */
double diff = distance_hausdorff(letters[i], contours[maxIndex]);
if (diff < lowestDiff) {
lowestDiff = diff;
best = 'a' + i;
}
}
if (lowestDiff > DIFF_THRESH) { // Dust
best = 0;
}
cout << best << " | diff: " << lowestDiff << endl;
// Show majority of last letters captured
letterCount %= NUM_LAST_LETTERS;
lastLetters[letterCount++] = best;
letterText = Mat::zeros(200, 200, CV_8UC3);
int counts[MAX_WORDS+1] = {0};
for (int i = 0; i < NUM_LAST_LETTERS; i++)
counts[lastLetters[i] + 1 - 'a']++;
int maxCount = 0;
char maxChar = 0;
for (int i = 0; i < MAX_WORDS+1; i++) {
if (counts[i] > maxCount) {
maxCount = counts[i];
maxChar = i;
}
}
if (maxChar && maxCount >= MIN_FREQ) {
maxChar = maxChar - 1 + 'a';
char buf[2 * sizeof(char)];
sprintf(buf, "%c", maxChar);
putText(letterText, buf, Point(10, 75), CV_FONT_NORMAL, 3, Scalar(255, 255, 255), 1, 1);
vector<vector<Point> > dummy;
dummy.push_back(letters[maxChar-'a']);
drawContours( letterText, dummy, 0, Scalar(255, 0, 0), 2, 8, hierarchy, 0, Point(0, 0) );
if (maxChar != lastExecLetter) {
lastExecLetter = maxChar;
doSystemCalls(maxChar);
}
}
}
// Show the current frame and the fg masks
imshow("Crop Frame", cropFrame);
imshow("Foreground", drawing);
if (contourImg.rows > 0)
imshow("Contour", contourImg);
imshow("Letter", letterText);
// Get the input from the keyboard
keyboard = waitKey(1);
// Save image as keyboard input
if (keyboard >= 'a' && keyboard <= 'z') {
cout << "Wrote letter '" << (char)keyboard << '\'' << endl;
// save in memory
letters[keyboard - 'a'] = contours[maxIndex];
// write to file
char buf[13 * sizeof(char)];
sprintf(buf, "images/%c.png", (char)keyboard);
imwrite(buf, drawing);
}
// Manual reset
if (keyboard == ' ')
pMOG2 = createBackgroundSubtractorMOG2(2000);
}
// Delete capture object
capture.release();
}