Filtering feature

emscripten/ARBindEM.cpp

@@ -144,4 +144,9 @@ EMSCRIPTEN_BINDINGS(constant_bindings) {
 	constant("AR_MARKER_INFO_CUTOFF_PHASE_POSE_ERROR", AR_MARKER_INFO_CUTOFF_PHASE_POSE_ERROR + 0);
 	constant("AR_MARKER_INFO_CUTOFF_PHASE_POSE_ERROR_MULTI", AR_MARKER_INFO_CUTOFF_PHASE_POSE_ERROR_MULTI + 0);
 	constant("AR_MARKER_INFO_CUTOFF_PHASE_HEURISTIC_TROUBLESOME_MATRIX_CODES", AR_MARKER_INFO_CUTOFF_PHASE_HEURISTIC_TROUBLESOME_MATRIX_CODES + 0);
+
+	/* Filter Trans Mat defaults for arFilterTransMatInit */
+	constant("AR_FILTER_TRANS_MAT_SAMPLE_RATE_DEFAULT", AR_FILTER_TRANS_MAT_SAMPLE_RATE_DEFAULT);
+	constant("AR_FILTER_TRANS_MAT_CUTOFF_FREQ_DEFAULT", AR_FILTER_TRANS_MAT_CUTOFF_FREQ_DEFAULT);
+
 }

emscripten/ARToolKitJS.cpp

@@ -54,6 +54,12 @@ struct arController {
 	KpmHandle* kpmHandle;
 	AR2HandleT* ar2Handle;
 
+	#if WITH_FILTERING
+	ARFilterTransMatInfo *ftmi;
+	ARdouble   filterCutoffFrequency = 60.0;
+	ARdouble   filterSampleRate = 120.0;
+	#endif
+
 	int detectedPage = -2;  // -2 Tracking not inited, -1 tracking inited OK, >= 0 tracking online on page.
 
 	int surfaceSetCount = 0; // Running NFT marker id
@@ -96,6 +102,22 @@ extern "C" {
 		NFT API bindings
 	*/
 
+	void matrixLerp(ARdouble src[3][4], ARdouble dst[3][4], float interpolationFactor) {
+		for (int i=0; i<3; i++) {
+			for (int j=0; j<4; j++) {
+				dst[i][j] = (1 - interpolationFactor) * src[i][j] + interpolationFactor * dst[i][j];
+			}
+		}
+	}
+
+	void matrixLerp2(ARdouble src[3][4], ARdouble dst[3][4], float interpolationFactor) {
+		for (int i=0; i<3; i++) {
+			for (int j=0; j<4; j++) {
+				dst[i][j] = dst[i][j] + (src[i][j] - dst[i][j]) / interpolationFactor;
+			}
+		}
+	}
+
 	int getNFTMarkerInfo(int id, int markerIndex) {
 		if (arControllers.find(id) == arControllers.end()) { return ARCONTROLLER_NOT_FOUND; }
 		arController *arc = &(arControllers[id]);
@@ -108,10 +130,22 @@ extern "C" {
 		int kpmResultNum = -1;
 
 		float trans[3][4];
+
+		#if WITH_FILTERING
+		ARdouble transF[3][4];
+		ARdouble transFLerp[3][4];
+		memset( transFLerp, 0, 3 * 4 * sizeof(ARdouble) );
+		#endif
+
 		float err = -1;
 		if (arc->detectedPage == -2) {
 			kpmMatching( arc->kpmHandle, arc->videoLuma );
 			kpmGetResult( arc->kpmHandle, &kpmResult, &kpmResultNum );
+
+			#if WITH_FILTERING
+			arc->ftmi = arFilterTransMatInit(arc->filterSampleRate, arc->filterCutoffFrequency);
+			#endif
+
 			int i, j, k;
 			int flag = -1;
 			for( i = 0; i < kpmResultNum; i++ ) {
@@ -139,6 +173,28 @@ extern "C" {
 
 		if (arc->detectedPage >= 0) {
 			int trackResult = ar2TrackingMod(arc->ar2Handle, arc->surfaceSet[arc->detectedPage], arc->videoFrame, trans, &err);
+
+      #if WITH_FILTERING
+			for (int j = 0; j < 3; j++) {
+				for (int k = 0; k < 4; k++) {
+					transF[j][k] = trans[j][k];
+				}
+			}
+
+			bool reset;
+			if (trackResult < 0) {
+				reset = 1;
+			} else {
+				reset = 0;
+			}
+
+			if (arFilterTransMat(arc->ftmi, transF, reset) < 0) {
+					ARLOGe("arFilterTransMat error with marker %d.\n", markerIndex);
+			}
+
+			matrixLerp2(transF, transFLerp, 0.95);
+			#endif
+
 			if( trackResult < 0 ) {
 				ARLOGi("Tracking lost. %d\n", trackResult);
 				arc->detectedPage = -2;
@@ -179,6 +235,25 @@ extern "C" {
 				markerIndex,
 				err,
 
+        #if WITH_FILTERING
+
+				transFLerp[0][0],
+				transFLerp[0][1],
+				transFLerp[0][2],
+				transFLerp[0][3],
+
+				transFLerp[1][0],
+				transFLerp[1][1],
+				transFLerp[1][2],
+				transFLerp[1][3],
+
+				transFLerp[2][0],
+				transFLerp[2][1],
+				transFLerp[2][2],
+				transFLerp[2][3]
+
+				#else
+
 				trans[0][0],
 				trans[0][1],
 				trans[0][2],
@@ -193,6 +268,8 @@ extern "C" {
 				trans[2][1],
 				trans[2][2],
 				trans[2][3]
+
+				#endif
 			);
         } else {
 			EM_ASM_({
@@ -347,6 +424,13 @@ extern "C" {
 			arPattDetach(arc->arhandle);
 			arDeleteHandle(arc->arhandle);
 			arc->arhandle = NULL;
+
+			#if WITH_FILTERING
+			if (arc->ftmi) {
+					arFilterTransMatFinal(arc->ftmi);
+					arc->ftmi = NULL;
+			}
+			#endif
 		}
 		if (arc->ar3DHandle != NULL) {
 			ar3DDeleteHandle(&(arc->ar3DHandle));

emscripten/NaiveFilter.h

@@ -0,0 +1,32 @@
+#pragma once
+
+// This implementation "inspired" by a questionable interpretation of a Kalman Filter.
+template<typename PointT>
+class NaiveFilter
+{
+public:
+    NaiveFilter(PointT initialEstimate, double lambda)
+        : m_lastEstimate{ initialEstimate }
+        , m_lambda{ lambda }
+    {}
+
+    void Update(PointT& measurement, double deltaT)
+    {
+        auto measurementDelta = measurement - m_lastEstimate;
+        double measurementError = measurementDelta.dot(measurementDelta);
+
+        double gain = (m_lastEstimateError + EPSILON) / (m_lastEstimateError + measurementError + EPSILON);
+
+        m_lastEstimate += gain * measurementDelta;
+        m_lastEstimateError = std::max((1.f - gain) * m_lastEstimateError, m_lambda * deltaT * measurementError);
+
+        measurement = m_lastEstimate;
+    }
+
+private:
+    static constexpr double EPSILON = 0.00001; // To prevent gain from becoming NaN.
+    const double m_lambda{};
+
+    PointT m_lastEstimate{};
+    double m_lastEstimateError{ 10.f };
+};

examples/nft_improved_worker/index.html

@@ -10,9 +10,12 @@ <h3>jsartoolkit5 demos with ar2Tracking</h3>
     <div>Image to recognize for the following examples <a href="https://imgur.com/HvcmRVl">https://imgur.com/HvcmRVl</a></div>
     <p>Examples:</p>
     <ul>
-        <li><a href="threejs_worker_gltf.html">Show a 3D, animated model. Uses web worker. Has loader. Has interpolation.</a></li>
+        <li><a href="threejs_worker_gltf.html">Show a 3D animated model. Uses web worker. Has loader. Has interpolation.</a></li>
         <li><a href="main_threejs_worker.html">Show a 3D sphere. Uses web worker. Has loader. Has interpolation. Asm version. (high framerate)</a></li>
+        <li><a href="main_threejs_filter_worker.html">Show a 3D sphere. Uses web worker. Has loader. No js interpolation, with filtering and matrixLerp. Asm version. (high framerate)</a></li>
+        <li><a href="main_threejs_filter_worker_gltf.html">Show a 3D animated model. Uses web worker. Has loader. No js interpolation, with filtering and matrixLerp. Asm version. (high framerate)</a></li>
         <li><a href="main_threejs_wasm_worker.html">Show a 3D sphere. Uses web worker. Has loader. Has interpolation. Uses web assembly</a></li>
+        <li><a href="main_threejs_filter_wasm_worker.html">Show a 3D sphere. Uses web worker. Has loader. No js interpolation, with filtering. Uses web assembly</a></li>
         <li><a href="main.html">Draw a red rectangle around the image. Single-threaded. No loader</a></li>
         <li><a href="main_worker.html">Draw a red rectangle around the image. Uses web worker. No loader</a></li>
     </ul>

examples/nft_improved_worker/main_threejs_filter_wasm_worker.html

@@ -0,0 +1,140 @@
+<!DOCTYPE html>
+<html>
+<head>
+	<meta charset="utf-8">
+	<title>NFT marker example with a WebWorker, WASM and with Three.js</title>
+	<meta name="viewport" content="width=device-width, initial-scale=1, minimum-scale=0.5, maximum-scale=1">
+	<link rel="stylesheet" href="../css/video-style.css">
+</head>
+<body class="loading">
+	<div id="loading" >
+			<img  src="../Data/logo.gif"/>
+			<span class="loading-text">Loading, please wait</span>
+	</div>
+  <!--
+  ==================
+  STATS
+  ==================
+  -->
+
+      <div id="stats" class="ui stats">
+
+          <div id="stats1" class="stats-item">
+              <p class="stats-item-title">
+                  Main
+              </p>
+          </div>
+
+          <div id="stats2" class="stats-item">
+              <p class="stats-item-title">
+                  Worker
+              </p>
+          </div>
+
+      </div>
+
+  <!--
+  ==================
+  CAMERA VIDEO & CANVAS
+  ==================
+  -->
+
+      <div id="app">
+          <video
+              loop
+              autoplay
+              muted
+              playsinline
+              id="video">
+          </video>
+
+          <canvas id="canvas"></canvas>
+      </div>
+
+      <a
+          href="https://raw.githubusercontent.com/artoolkit/artoolkit5/master/doc/Marker%20images/pinball.jpg"
+          class="ui marker"
+          target="_blank">
+          🖼 Marker Image
+      </a>
+
+<script src="../js/third_party/three.js/stats.min.js"></script>
+<script src="../js/third_party/three.js/three.min.js"></script>
+<script src="threejs_wasm_filter_worker.js"></script>
+
+<script>
+
+/**
+ * STATS
+ */
+var statsMain = new Stats();
+statsMain.showPanel( 0 ); // 0: fps, 1: ms, 2: mb, 3+: custom
+document.getElementById( 'stats1' ).appendChild( statsMain.dom );
+
+var statsWorker = new Stats();
+statsWorker.showPanel( 0 ); // 0: fps, 1: ms, 2: mb, 3+: custom
+document.getElementById( 'stats2' ).appendChild( statsWorker.dom );
+
+/**
+ * APP / ELEMENTS
+ */
+var container = document.getElementById( 'app' );
+var video = document.getElementById( 'video' );
+var canvas = document.getElementById( 'canvas' );
+
+/**
+ * APP / VIDEO STREAM
+ */
+
+		if (navigator.mediaDevices && navigator.mediaDevices.getUserMedia) {
+			var hint = {
+					audio: false,
+					video: true
+			};
+			if( window.innerWidth < 800 ) {
+					var width = ( window.innerWidth < window.innerHeight ) ? 240 : 360;
+					var height = ( window.innerWidth < window.innerHeight ) ? 360 : 240;
+
+					var aspectRatio = window.innerWidth / window.innerHeight;
+
+					console.log( width, height );
+
+					hint = {
+							audio: false,
+							video: {
+									facingMode: 'environment',
+									width: { min: width, max: width }
+							},
+					};
+
+					console.log( hint );
+			}
+
+			navigator.mediaDevices.getUserMedia( hint ).then( function( stream ) {
+					video.srcObject = stream;
+					video.addEventListener( 'loadedmetadata', function() {
+							video.play();
+
+							console.log( 'video', video, video.videoWidth, video.videoHeight );
+
+							start(
+									container,
+									markers['pinball'],
+									video,
+									video.videoWidth,
+									video.videoHeight,
+									canvas,
+									function() {
+											statsMain.update()
+									},
+									function() {
+											statsWorker.update();
+									},
+									null
+							);
+					} );
+			} );
+	}
+</script>
+</body>
+</html>