diff --git a/README.md b/README.md index 1e0db2e..317eb49 100755 --- a/README.md +++ b/README.md @@ -86,7 +86,7 @@ We've included caffe pre-trained models. Should you use these pre-trained weight [![Predicted flows on MPI-Sintel](./image.png)](https://www.youtube.com/watch?v=HtBmabY8aeU "Predicted flows on MPI-Sintel") ## Reference -If you find this implementation useful in your work, please acknowledge it appropriately and cite the paper using: +If you find this implementation useful in your work, please acknowledge it appropriately and cite the paper: ```` @InProceedings{IMKDB17, author = "E. Ilg and N. Mayer and T. Saikia and M. Keuper and A. Dosovitskiy and T. Brox", @@ -97,6 +97,19 @@ If you find this implementation useful in your work, please acknowledge it appro url = "http://lmb.informatik.uni-freiburg.de//Publications/2017/IMKDB17" } ```` +``` +@misc{flownet2-pytorch, + author = {Fitsum Reda and Robert Pottorff and Jon Barker and Bryan Catanzaro}, + title = {flownet2-pytorch: Pytorch implementation of FlowNet 2.0: Evolution of Optical Flow Estimation with Deep Networks}, + year = {2017}, + publisher = {GitHub}, + journal = {GitHub repository}, + howpublished = {\url{https://github.com/NVIDIA/flownet2-pytorch}} +} +``` +## Related Optical Flow Work from Nvidia +Code (in Caffe and Pytorch): [PWC-Net](https://github.com/NVlabs/PWC-Net)
+Paper : [PWC-Net: CNNs for Optical Flow Using Pyramid, Warping, and Cost Volume](https://arxiv.org/abs/1709.02371). ## Acknowledgments Parts of this code were derived, as noted in the code, from [ClementPinard/FlowNetPytorch](https://github.com/ClementPinard/FlowNetPytorch). diff --git a/models.py b/models.py index 6ec7f32..7ac8a58 100755 --- a/models.py +++ b/models.py @@ -107,23 +107,27 @@ def init_deconv_bilinear(self, weight): return def forward(self, inputs): + # input format: 5d tensor, (batch, rgb, img #, x, y) + # normalizing imgs rgb_mean = inputs.contiguous().view(inputs.size()[:2]+(-1,)).mean(dim=-1).view(inputs.size()[:2] + (1,1,1,)) - x = (inputs - rgb_mean) / self.rgb_max + + # squish the img # dimension by concatenating the rgb channels x1 = x[:,:,0,:,:] x2 = x[:,:,1,:,:] x = torch.cat((x1,x2), dim = 1) # flownetc - flownetc_flow2 = self.flownetc(x)[0] + flownetc_flow2 = self.flownetc(x)[0] # only return flow2, since the network returns flow6 to flow2 flownetc_flow = self.upsample1(flownetc_flow2*self.div_flow) - # warp img1 to img0; magnitude of diff between img0 and and warped_img1, + # warp img1 to img0; magnitude of diff between img0 and and warped_img1, i.e. brightness error + # combine the flow with img1 to 'reconstruct' img0 resampled_img1 = self.resample1(x[:,3:,:,:], flownetc_flow) - diff_img0 = x[:,:3,:,:] - resampled_img1 + diff_img0 = x[:,:3,:,:] - resampled_img1 # brightness error norm_diff_img0 = self.channelnorm(diff_img0) - # concat img0, img1, img1->img0, flow, diff-mag ; + # concat img0, img1, warped img1, flow, normalized brightness error concat1 = torch.cat((x, resampled_img1, flownetc_flow/self.div_flow, norm_diff_img0), dim=1) # flownets1 diff --git a/networks/channelnorm_package/build.py b/networks/channelnorm_package/build.py index 2066819..a9be1cd 100755 --- a/networks/channelnorm_package/build.py +++ b/networks/channelnorm_package/build.py @@ -24,8 +24,9 @@ package=False, relative_to=this_folder, define_macros=Defines, - extra_objects=[os.path.join(this_folder, Object) for Object in Objects] + extra_objects=[os.path.join(this_folder, Object) for Object in Objects], + extra_compile_args=['-std=c99'] ) if __name__ == '__main__': - ffi.build() \ No newline at end of file + ffi.build() diff --git a/networks/channelnorm_package/functions/channelnorm.py b/networks/channelnorm_package/functions/channelnorm.py index f84bc31..a4c19a6 100755 --- a/networks/channelnorm_package/functions/channelnorm.py +++ b/networks/channelnorm_package/functions/channelnorm.py @@ -12,7 +12,8 @@ def forward(ctx, input1, norm_deg=2): channelnorm.ChannelNorm_cuda_forward(input1, output, norm_deg) ctx.save_for_backward(input1, output) - + ctx.norm_deg = norm_deg + return output @staticmethod diff --git a/networks/correlation_package/build.py b/networks/correlation_package/build.py index 9974e2a..c86dd76 100755 --- a/networks/correlation_package/build.py +++ b/networks/correlation_package/build.py @@ -24,7 +24,8 @@ package=False, relative_to=this_folder, define_macros=Defines, - extra_objects=[os.path.join(this_folder, Object) for Object in Objects] + extra_objects=[os.path.join(this_folder, Object) for Object in Objects], + extra_compile_args=['-std=c99'] ) if __name__ == '__main__': diff --git a/networks/resample2d_package/build.py b/networks/resample2d_package/build.py index f1de130..7ac66c7 100755 --- a/networks/resample2d_package/build.py +++ b/networks/resample2d_package/build.py @@ -24,8 +24,9 @@ package=False, relative_to=this_folder, define_macros=Defines, - extra_objects=[os.path.join(this_folder, Object) for Object in Objects] + extra_objects=[os.path.join(this_folder, Object) for Object in Objects], + extra_compile_args=['-std=c99'] ) if __name__ == '__main__': - ffi.build() \ No newline at end of file + ffi.build() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..825c162 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +setproctitle==1.1.10 +tensorboardX==1.2 +colorama==0.3.9