diff --git a/Cargo.toml b/Cargo.toml index 8d1ec616..b7ef4928 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "feroxbuster" -version = "1.12.0" +version = "1.12.1" authors = ["Ben 'epi' Risher "] license = "MIT" edition = "2018" diff --git a/src/filters.rs b/src/filters.rs deleted file mode 100644 index e32bed8e..00000000 --- a/src/filters.rs +++ /dev/null @@ -1,513 +0,0 @@ -use crate::config::CONFIGURATION; -use crate::utils::get_url_path_length; -use crate::{FeroxResponse, FeroxSerialize}; -use fuzzyhash::FuzzyHash; -use regex::Regex; -use std::any::Any; -use std::fmt::Debug; - -// references: -// https://dev.to/magnusstrale/rust-trait-objects-in-a-vector-non-trivial-4co5 -// https://stackoverflow.com/questions/25339603/how-to-test-for-equality-between-trait-objects - -/// FeroxFilter trait; represents different types of possible filters that can be applied to -/// responses -pub trait FeroxFilter: Debug + Send + Sync { - /// Determine whether or not this particular filter should be applied or not - fn should_filter_response(&self, response: &FeroxResponse) -> bool; - - /// delegates to the FeroxFilter-implementing type which gives us the actual type of self - fn box_eq(&self, other: &dyn Any) -> bool; - - /// gives us `other` as Any in box_eq - fn as_any(&self) -> &dyn Any; -} - -/// implementation of PartialEq, necessary long-form due to "trait cannot be made into an object" -/// error when attempting to derive PartialEq on the trait itself -impl PartialEq for Box { - /// Perform a comparison of two implementors of the FeroxFilter trait - fn eq(&self, other: &Box) -> bool { - self.box_eq(other.as_any()) - } -} - -/// Data holder for two pieces of data needed when auto-filtering out wildcard responses -/// -/// `dynamic` is the size of the response that will later be combined with the length -/// of the path of the url requested and used to determine interesting pages from custom -/// 404s where the requested url is reflected back in the response -/// -/// `size` is size of the response that should be included with filters passed via runtime -/// configuration and any static wildcard lengths. -#[derive(Debug, Default, Clone, PartialEq)] -pub struct WildcardFilter { - /// size of the response that will later be combined with the length of the path of the url - /// requested - pub dynamic: u64, - - /// size of the response that should be included with filters passed via runtime configuration - pub size: u64, -} - -/// implementation of FeroxFilter for WildcardFilter -impl FeroxFilter for WildcardFilter { - /// Examine size, dynamic, and content_len to determine whether or not the response received - /// is a wildcard response and therefore should be filtered out - fn should_filter_response(&self, response: &FeroxResponse) -> bool { - log::trace!("enter: should_filter_response({:?} {})", self, response); - - // quick return if dont_filter is set - if CONFIGURATION.dont_filter { - // --dont-filter applies specifically to wildcard filters, it is not a 100% catch all - // for not filtering anything. As such, it should live in the implementation of - // a wildcard filter - return false; - } - - if self.size > 0 && self.size == response.content_length() { - // static wildcard size found during testing - // size isn't default, size equals response length, and auto-filter is on - log::debug!("static wildcard: filtered out {}", response.url()); - log::trace!("exit: should_filter_response -> true"); - return true; - } - - if self.dynamic > 0 { - // dynamic wildcard offset found during testing - - // I'm about to manually split this url path instead of using reqwest::Url's - // builtin parsing. The reason is that they call .split() on the url path - // except that I don't want an empty string taking up the last index in the - // event that the url ends with a forward slash. It's ugly enough to be split - // into its own function for readability. - let url_len = get_url_path_length(&response.url()); - - if url_len + self.dynamic == response.content_length() { - log::debug!("dynamic wildcard: filtered out {}", response.url()); - log::trace!("exit: should_filter_response -> true"); - return true; - } - } - log::trace!("exit: should_filter_response -> false"); - false - } - - /// Compare one WildcardFilter to another - fn box_eq(&self, other: &dyn Any) -> bool { - other.downcast_ref::().map_or(false, |a| self == a) - } - - /// Return self as Any for dynamic dispatch purposes - fn as_any(&self) -> &dyn Any { - self - } -} - -/// Simple implementor of FeroxFilter; used to filter out status codes specified using -/// -C|--filter-status -#[derive(Default, Debug, PartialEq)] -pub struct StatusCodeFilter { - /// Status code that should not be displayed to the user - pub filter_code: u16, -} - -/// implementation of FeroxFilter for StatusCodeFilter -impl FeroxFilter for StatusCodeFilter { - /// Check `filter_code` against what was passed in via -C|--filter-status - fn should_filter_response(&self, response: &FeroxResponse) -> bool { - log::trace!("enter: should_filter_response({:?} {})", self, response); - - if response.status().as_u16() == self.filter_code { - log::debug!( - "filtered out {} based on --filter-status of {}", - response.url(), - self.filter_code - ); - log::trace!("exit: should_filter_response -> true"); - return true; - } - - log::trace!("exit: should_filter_response -> false"); - false - } - - /// Compare one StatusCodeFilter to another - fn box_eq(&self, other: &dyn Any) -> bool { - other.downcast_ref::().map_or(false, |a| self == a) - } - - /// Return self as Any for dynamic dispatch purposes - fn as_any(&self) -> &dyn Any { - self - } -} - -/// Simple implementor of FeroxFilter; used to filter out responses based on the number of lines -/// in a Response body; specified using -N|--filter-lines -#[derive(Default, Debug, PartialEq)] -pub struct LinesFilter { - /// Number of lines in a Response's body that should be filtered - pub line_count: usize, -} - -/// implementation of FeroxFilter for LinesFilter -impl FeroxFilter for LinesFilter { - /// Check `line_count` against what was passed in via -N|--filter-lines - fn should_filter_response(&self, response: &FeroxResponse) -> bool { - log::trace!("enter: should_filter_response({:?} {})", self, response); - - let result = response.line_count() == self.line_count; - - log::trace!("exit: should_filter_response -> {}", result); - - result - } - - /// Compare one LinesFilter to another - fn box_eq(&self, other: &dyn Any) -> bool { - other.downcast_ref::().map_or(false, |a| self == a) - } - - /// Return self as Any for dynamic dispatch purposes - fn as_any(&self) -> &dyn Any { - self - } -} - -/// Simple implementor of FeroxFilter; used to filter out responses based on the number of words -/// in a Response body; specified using -W|--filter-words -#[derive(Default, Debug, PartialEq)] -pub struct WordsFilter { - /// Number of words in a Response's body that should be filtered - pub word_count: usize, -} - -/// implementation of FeroxFilter for WordsFilter -impl FeroxFilter for WordsFilter { - /// Check `word_count` against what was passed in via -W|--filter-words - fn should_filter_response(&self, response: &FeroxResponse) -> bool { - log::trace!("enter: should_filter_response({:?} {})", self, response); - - let result = response.word_count() == self.word_count; - - log::trace!("exit: should_filter_response -> {}", result); - - result - } - - /// Compare one WordsFilter to another - fn box_eq(&self, other: &dyn Any) -> bool { - other.downcast_ref::().map_or(false, |a| self == a) - } - - /// Return self as Any for dynamic dispatch purposes - fn as_any(&self) -> &dyn Any { - self - } -} - -/// Simple implementor of FeroxFilter; used to filter out responses based on the length of a -/// Response body; specified using -S|--filter-size -#[derive(Default, Debug, PartialEq)] -pub struct SizeFilter { - /// Overall length of a Response's body that should be filtered - pub content_length: u64, -} - -/// implementation of FeroxFilter for SizeFilter -impl FeroxFilter for SizeFilter { - /// Check `content_length` against what was passed in via -S|--filter-size - fn should_filter_response(&self, response: &FeroxResponse) -> bool { - log::trace!("enter: should_filter_response({:?} {})", self, response); - - let result = response.content_length() == self.content_length; - - log::trace!("exit: should_filter_response -> {}", result); - - result - } - - /// Compare one SizeFilter to another - fn box_eq(&self, other: &dyn Any) -> bool { - other.downcast_ref::().map_or(false, |a| self == a) - } - - /// Return self as Any for dynamic dispatch purposes - fn as_any(&self) -> &dyn Any { - self - } -} - -/// Simple implementor of FeroxFilter; used to filter out responses based on a given regular -/// expression; specified using -X|--filter-regex -#[derive(Debug)] -pub struct RegexFilter { - /// Regular expression to be applied to the response body for filtering, compiled - pub compiled: Regex, - - /// Regular expression as passed in on the command line, not compiled - pub raw_string: String, -} - -/// implementation of FeroxFilter for RegexFilter -impl FeroxFilter for RegexFilter { - /// Check `expression` against the response body, if the expression matches, the response - /// should be filtered out - fn should_filter_response(&self, response: &FeroxResponse) -> bool { - log::trace!("enter: should_filter_response({:?} {})", self, response); - - let result = self.compiled.is_match(response.text()); - - log::trace!("exit: should_filter_response -> {}", result); - - result - } - - /// Compare one SizeFilter to another - fn box_eq(&self, other: &dyn Any) -> bool { - other.downcast_ref::().map_or(false, |a| self == a) - } - - /// Return self as Any for dynamic dispatch purposes - fn as_any(&self) -> &dyn Any { - self - } -} - -/// PartialEq implementation for RegexFilter -impl PartialEq for RegexFilter { - /// Simple comparison of the raw string passed in via the command line - fn eq(&self, other: &RegexFilter) -> bool { - self.raw_string == other.raw_string - } -} - -/// Simple implementor of FeroxFilter; used to filter out responses based on the similarity of a -/// Response body with a known response; specified using --filter-similar-to -#[derive(Default, Debug, PartialEq)] -pub struct SimilarityFilter { - /// Response's body to be used for comparison for similarity - pub text: String, - - /// Percentage of similarity at which a page is determined to be a near-duplicate of another - pub threshold: u32, -} - -/// implementation of FeroxFilter for SimilarityFilter -impl FeroxFilter for SimilarityFilter { - /// Check `FeroxResponse::text` against what was requested from the site passed in via - /// --filter-similar-to - fn should_filter_response(&self, response: &FeroxResponse) -> bool { - let other = FuzzyHash::new(&response.text); - - if let Ok(result) = FuzzyHash::compare(&self.text, &other.to_string()) { - return result >= self.threshold; - } - - // couldn't hash the response, don't filter - log::warn!("Could not hash body from {}", response.as_str()); - false - } - - /// Compare one SimilarityFilter to another - fn box_eq(&self, other: &dyn Any) -> bool { - other.downcast_ref::().map_or(false, |a| self == a) - } - - /// Return self as Any for dynamic dispatch purposes - fn as_any(&self) -> &dyn Any { - self - } -} - -#[cfg(test)] -mod tests { - use super::*; - use reqwest::Url; - - #[test] - /// just a simple test to increase code coverage by hitting as_any and the inner value - fn lines_filter_as_any() { - let filter = LinesFilter { line_count: 1 }; - - assert_eq!(filter.line_count, 1); - assert_eq!( - *filter.as_any().downcast_ref::().unwrap(), - filter - ); - } - - #[test] - /// just a simple test to increase code coverage by hitting as_any and the inner value - fn words_filter_as_any() { - let filter = WordsFilter { word_count: 1 }; - - assert_eq!(filter.word_count, 1); - assert_eq!( - *filter.as_any().downcast_ref::().unwrap(), - filter - ); - } - - #[test] - /// just a simple test to increase code coverage by hitting as_any and the inner value - fn size_filter_as_any() { - let filter = SizeFilter { content_length: 1 }; - - assert_eq!(filter.content_length, 1); - assert_eq!( - *filter.as_any().downcast_ref::().unwrap(), - filter - ); - } - - #[test] - /// just a simple test to increase code coverage by hitting as_any and the inner value - fn status_code_filter_as_any() { - let filter = StatusCodeFilter { filter_code: 200 }; - - assert_eq!(filter.filter_code, 200); - assert_eq!( - *filter.as_any().downcast_ref::().unwrap(), - filter - ); - } - - #[test] - /// just a simple test to increase code coverage by hitting as_any and the inner value - fn regex_filter_as_any() { - let raw = r".*\.txt$"; - let compiled = Regex::new(raw).unwrap(); - let filter = RegexFilter { - compiled, - raw_string: raw.to_string(), - }; - - assert_eq!(filter.raw_string, r".*\.txt$"); - assert_eq!( - *filter.as_any().downcast_ref::().unwrap(), - filter - ); - } - - #[test] - /// test should_filter on WilcardFilter where static logic matches - fn wildcard_should_filter_when_static_wildcard_found() { - let resp = FeroxResponse { - text: String::new(), - wildcard: true, - url: Url::parse("http://localhost").unwrap(), - content_length: 100, - word_count: 50, - line_count: 25, - headers: reqwest::header::HeaderMap::new(), - status: reqwest::StatusCode::OK, - }; - - let filter = WildcardFilter { - size: 100, - dynamic: 0, - }; - - assert!(filter.should_filter_response(&resp)); - } - - #[test] - /// test should_filter on WilcardFilter where dynamic logic matches - fn wildcard_should_filter_when_dynamic_wildcard_found() { - let resp = FeroxResponse { - text: String::new(), - wildcard: true, - url: Url::parse("http://localhost/stuff").unwrap(), - content_length: 100, - word_count: 50, - line_count: 25, - headers: reqwest::header::HeaderMap::new(), - status: reqwest::StatusCode::OK, - }; - - let filter = WildcardFilter { - size: 0, - dynamic: 95, - }; - - assert!(filter.should_filter_response(&resp)); - } - - #[test] - /// test should_filter on RegexFilter where regex matches body - fn regexfilter_should_filter_when_regex_matches_on_response_body() { - let resp = FeroxResponse { - text: String::from("im a body response hurr durr!"), - wildcard: false, - url: Url::parse("http://localhost/stuff").unwrap(), - content_length: 100, - word_count: 50, - line_count: 25, - headers: reqwest::header::HeaderMap::new(), - status: reqwest::StatusCode::OK, - }; - - let raw = r"response...rr"; - - let filter = RegexFilter { - raw_string: raw.to_string(), - compiled: Regex::new(raw).unwrap(), - }; - - assert!(filter.should_filter_response(&resp)); - } - - #[test] - /// a few simple tests for similarity filter - fn similarity_filter_is_accurate() { - let mut resp = FeroxResponse { - text: String::from("sitting"), - wildcard: false, - url: Url::parse("http://localhost/stuff").unwrap(), - content_length: 100, - word_count: 50, - line_count: 25, - headers: reqwest::header::HeaderMap::new(), - status: reqwest::StatusCode::OK, - }; - - let mut filter = SimilarityFilter { - text: FuzzyHash::new("kitten").to_string(), - threshold: 95, - }; - - // kitten/sitting is 57% similar, so a threshold of 95 should not be filtered - assert!(!filter.should_filter_response(&resp)); - - resp.text = String::new(); - filter.text = String::new(); - filter.threshold = 100; - - // two empty strings are the same, however ssdeep doesn't accept empty strings, expect false - assert!(!filter.should_filter_response(&resp)); - - resp.text = String::from("some data to hash for the purposes of running a test"); - filter.text = - FuzzyHash::new("some data to hash for the purposes of running a te").to_string(); - filter.threshold = 17; - - assert!(filter.should_filter_response(&resp)); - } - - #[test] - /// just a simple test to increase code coverage by hitting as_any and the inner value - fn similarity_filter_as_any() { - let filter = SimilarityFilter { - text: String::from("stuff"), - threshold: 95, - }; - - assert_eq!(filter.text, "stuff"); - assert_eq!( - *filter.as_any().downcast_ref::().unwrap(), - filter - ); - } -} diff --git a/src/filters/lines.rs b/src/filters/lines.rs new file mode 100644 index 00000000..1c5704dd --- /dev/null +++ b/src/filters/lines.rs @@ -0,0 +1,33 @@ +use super::*; + +/// Simple implementor of FeroxFilter; used to filter out responses based on the number of lines +/// in a Response body; specified using -N|--filter-lines +#[derive(Default, Debug, PartialEq)] +pub struct LinesFilter { + /// Number of lines in a Response's body that should be filtered + pub line_count: usize, +} + +/// implementation of FeroxFilter for LinesFilter +impl FeroxFilter for LinesFilter { + /// Check `line_count` against what was passed in via -N|--filter-lines + fn should_filter_response(&self, response: &FeroxResponse) -> bool { + log::trace!("enter: should_filter_response({:?} {})", self, response); + + let result = response.line_count() == self.line_count; + + log::trace!("exit: should_filter_response -> {}", result); + + result + } + + /// Compare one LinesFilter to another + fn box_eq(&self, other: &dyn Any) -> bool { + other.downcast_ref::().map_or(false, |a| self == a) + } + + /// Return self as Any for dynamic dispatch purposes + fn as_any(&self) -> &dyn Any { + self + } +} diff --git a/src/filters/mod.rs b/src/filters/mod.rs new file mode 100644 index 00000000..30f8510f --- /dev/null +++ b/src/filters/mod.rs @@ -0,0 +1,24 @@ +//! module containing all of feroxbuster's filters +mod traits; +mod wildcard; +mod status_code; +mod words; +mod lines; +mod size; +mod regex; +mod similarity; +#[cfg(test)] +mod tests; + +pub use self::lines::LinesFilter; +pub use self::regex::RegexFilter; +pub use self::similarity::SimilarityFilter; +pub use self::size::SizeFilter; +pub use self::status_code::StatusCodeFilter; +pub use self::traits::FeroxFilter; +pub use self::wildcard::WildcardFilter; +pub use self::words::WordsFilter; + +use crate::{config::CONFIGURATION, utils::get_url_path_length, FeroxResponse, FeroxSerialize}; +use std::any::Any; +use std::fmt::Debug; diff --git a/src/filters/regex.rs b/src/filters/regex.rs new file mode 100644 index 00000000..08c48f0a --- /dev/null +++ b/src/filters/regex.rs @@ -0,0 +1,46 @@ +use super::*; +use ::regex::Regex; + +/// Simple implementor of FeroxFilter; used to filter out responses based on a given regular +/// expression; specified using -X|--filter-regex +#[derive(Debug)] +pub struct RegexFilter { + /// Regular expression to be applied to the response body for filtering, compiled + pub compiled: Regex, + + /// Regular expression as passed in on the command line, not compiled + pub raw_string: String, +} + +/// implementation of FeroxFilter for RegexFilter +impl FeroxFilter for RegexFilter { + /// Check `expression` against the response body, if the expression matches, the response + /// should be filtered out + fn should_filter_response(&self, response: &FeroxResponse) -> bool { + log::trace!("enter: should_filter_response({:?} {})", self, response); + + let result = self.compiled.is_match(response.text()); + + log::trace!("exit: should_filter_response -> {}", result); + + result + } + + /// Compare one SizeFilter to another + fn box_eq(&self, other: &dyn Any) -> bool { + other.downcast_ref::().map_or(false, |a| self == a) + } + + /// Return self as Any for dynamic dispatch purposes + fn as_any(&self) -> &dyn Any { + self + } +} + +/// PartialEq implementation for RegexFilter +impl PartialEq for RegexFilter { + /// Simple comparison of the raw string passed in via the command line + fn eq(&self, other: &RegexFilter) -> bool { + self.raw_string == other.raw_string + } +} diff --git a/src/filters/similarity.rs b/src/filters/similarity.rs new file mode 100644 index 00000000..215598d7 --- /dev/null +++ b/src/filters/similarity.rs @@ -0,0 +1,40 @@ +use super::*; +use fuzzyhash::FuzzyHash; + +/// Simple implementor of FeroxFilter; used to filter out responses based on the similarity of a +/// Response body with a known response; specified using --filter-similar-to +#[derive(Default, Debug, PartialEq)] +pub struct SimilarityFilter { + /// Response's body to be used for comparison for similarity + pub text: String, + + /// Percentage of similarity at which a page is determined to be a near-duplicate of another + pub threshold: u32, +} + +/// implementation of FeroxFilter for SimilarityFilter +impl FeroxFilter for SimilarityFilter { + /// Check `FeroxResponse::text` against what was requested from the site passed in via + /// --filter-similar-to + fn should_filter_response(&self, response: &FeroxResponse) -> bool { + let other = FuzzyHash::new(&response.text); + + if let Ok(result) = FuzzyHash::compare(&self.text, &other.to_string()) { + return result >= self.threshold; + } + + // couldn't hash the response, don't filter + log::warn!("Could not hash body from {}", response.as_str()); + false + } + + /// Compare one SimilarityFilter to another + fn box_eq(&self, other: &dyn Any) -> bool { + other.downcast_ref::().map_or(false, |a| self == a) + } + + /// Return self as Any for dynamic dispatch purposes + fn as_any(&self) -> &dyn Any { + self + } +} diff --git a/src/filters/size.rs b/src/filters/size.rs new file mode 100644 index 00000000..73ef3505 --- /dev/null +++ b/src/filters/size.rs @@ -0,0 +1,33 @@ +use super::*; + +/// Simple implementor of FeroxFilter; used to filter out responses based on the length of a +/// Response body; specified using -S|--filter-size +#[derive(Default, Debug, PartialEq)] +pub struct SizeFilter { + /// Overall length of a Response's body that should be filtered + pub content_length: u64, +} + +/// implementation of FeroxFilter for SizeFilter +impl FeroxFilter for SizeFilter { + /// Check `content_length` against what was passed in via -S|--filter-size + fn should_filter_response(&self, response: &FeroxResponse) -> bool { + log::trace!("enter: should_filter_response({:?} {})", self, response); + + let result = response.content_length() == self.content_length; + + log::trace!("exit: should_filter_response -> {}", result); + + result + } + + /// Compare one SizeFilter to another + fn box_eq(&self, other: &dyn Any) -> bool { + other.downcast_ref::().map_or(false, |a| self == a) + } + + /// Return self as Any for dynamic dispatch purposes + fn as_any(&self) -> &dyn Any { + self + } +} diff --git a/src/filters/status_code.rs b/src/filters/status_code.rs new file mode 100644 index 00000000..b462b89d --- /dev/null +++ b/src/filters/status_code.rs @@ -0,0 +1,40 @@ +use super::*; + +/// Simple implementor of FeroxFilter; used to filter out status codes specified using +/// -C|--filter-status +#[derive(Default, Debug, PartialEq)] +pub struct StatusCodeFilter { + /// Status code that should not be displayed to the user + pub filter_code: u16, +} + +/// implementation of FeroxFilter for StatusCodeFilter +impl FeroxFilter for StatusCodeFilter { + /// Check `filter_code` against what was passed in via -C|--filter-status + fn should_filter_response(&self, response: &FeroxResponse) -> bool { + log::trace!("enter: should_filter_response({:?} {})", self, response); + + if response.status().as_u16() == self.filter_code { + log::debug!( + "filtered out {} based on --filter-status of {}", + response.url(), + self.filter_code + ); + log::trace!("exit: should_filter_response -> true"); + return true; + } + + log::trace!("exit: should_filter_response -> false"); + false + } + + /// Compare one StatusCodeFilter to another + fn box_eq(&self, other: &dyn Any) -> bool { + other.downcast_ref::().map_or(false, |a| self == a) + } + + /// Return self as Any for dynamic dispatch purposes + fn as_any(&self) -> &dyn Any { + self + } +} diff --git a/src/filters/tests.rs b/src/filters/tests.rs new file mode 100644 index 00000000..fcbc245b --- /dev/null +++ b/src/filters/tests.rs @@ -0,0 +1,188 @@ +use super::*; +use ::fuzzyhash::FuzzyHash; +use ::regex::Regex; +use reqwest::Url; + +#[test] +/// just a simple test to increase code coverage by hitting as_any and the inner value +fn lines_filter_as_any() { + let filter = LinesFilter { line_count: 1 }; + + assert_eq!(filter.line_count, 1); + assert_eq!( + *filter.as_any().downcast_ref::().unwrap(), + filter + ); +} + +#[test] +/// just a simple test to increase code coverage by hitting as_any and the inner value +fn words_filter_as_any() { + let filter = WordsFilter { word_count: 1 }; + + assert_eq!(filter.word_count, 1); + assert_eq!( + *filter.as_any().downcast_ref::().unwrap(), + filter + ); +} + +#[test] +/// just a simple test to increase code coverage by hitting as_any and the inner value +fn size_filter_as_any() { + let filter = SizeFilter { content_length: 1 }; + + assert_eq!(filter.content_length, 1); + assert_eq!( + *filter.as_any().downcast_ref::().unwrap(), + filter + ); +} + +#[test] +/// just a simple test to increase code coverage by hitting as_any and the inner value +fn status_code_filter_as_any() { + let filter = StatusCodeFilter { filter_code: 200 }; + + assert_eq!(filter.filter_code, 200); + assert_eq!( + *filter.as_any().downcast_ref::().unwrap(), + filter + ); +} + +#[test] +/// just a simple test to increase code coverage by hitting as_any and the inner value +fn regex_filter_as_any() { + let raw = r".*\.txt$"; + let compiled = Regex::new(raw).unwrap(); + let filter = RegexFilter { + compiled, + raw_string: raw.to_string(), + }; + + assert_eq!(filter.raw_string, r".*\.txt$"); + assert_eq!( + *filter.as_any().downcast_ref::().unwrap(), + filter + ); +} + +#[test] +/// test should_filter on WilcardFilter where static logic matches +fn wildcard_should_filter_when_static_wildcard_found() { + let resp = FeroxResponse { + text: String::new(), + wildcard: true, + url: Url::parse("http://localhost").unwrap(), + content_length: 100, + word_count: 50, + line_count: 25, + headers: reqwest::header::HeaderMap::new(), + status: reqwest::StatusCode::OK, + }; + + let filter = WildcardFilter { + size: 100, + dynamic: 0, + }; + + assert!(filter.should_filter_response(&resp)); +} + +#[test] +/// test should_filter on WilcardFilter where dynamic logic matches +fn wildcard_should_filter_when_dynamic_wildcard_found() { + let resp = FeroxResponse { + text: String::new(), + wildcard: true, + url: Url::parse("http://localhost/stuff").unwrap(), + content_length: 100, + word_count: 50, + line_count: 25, + headers: reqwest::header::HeaderMap::new(), + status: reqwest::StatusCode::OK, + }; + + let filter = WildcardFilter { + size: 0, + dynamic: 95, + }; + + assert!(filter.should_filter_response(&resp)); +} + +#[test] +/// test should_filter on RegexFilter where regex matches body +fn regexfilter_should_filter_when_regex_matches_on_response_body() { + let resp = FeroxResponse { + text: String::from("im a body response hurr durr!"), + wildcard: false, + url: Url::parse("http://localhost/stuff").unwrap(), + content_length: 100, + word_count: 50, + line_count: 25, + headers: reqwest::header::HeaderMap::new(), + status: reqwest::StatusCode::OK, + }; + + let raw = r"response...rr"; + + let filter = RegexFilter { + raw_string: raw.to_string(), + compiled: Regex::new(raw).unwrap(), + }; + + assert!(filter.should_filter_response(&resp)); +} + +#[test] +/// a few simple tests for similarity filter +fn similarity_filter_is_accurate() { + let mut resp = FeroxResponse { + text: String::from("sitting"), + wildcard: false, + url: Url::parse("http://localhost/stuff").unwrap(), + content_length: 100, + word_count: 50, + line_count: 25, + headers: reqwest::header::HeaderMap::new(), + status: reqwest::StatusCode::OK, + }; + + let mut filter = SimilarityFilter { + text: FuzzyHash::new("kitten").to_string(), + threshold: 95, + }; + + // kitten/sitting is 57% similar, so a threshold of 95 should not be filtered + assert!(!filter.should_filter_response(&resp)); + + resp.text = String::new(); + filter.text = String::new(); + filter.threshold = 100; + + // two empty strings are the same, however ssdeep doesn't accept empty strings, expect false + assert!(!filter.should_filter_response(&resp)); + + resp.text = String::from("some data to hash for the purposes of running a test"); + filter.text = FuzzyHash::new("some data to hash for the purposes of running a te").to_string(); + filter.threshold = 17; + + assert!(filter.should_filter_response(&resp)); +} + +#[test] +/// just a simple test to increase code coverage by hitting as_any and the inner value +fn similarity_filter_as_any() { + let filter = SimilarityFilter { + text: String::from("stuff"), + threshold: 95, + }; + + assert_eq!(filter.text, "stuff"); + assert_eq!( + *filter.as_any().downcast_ref::().unwrap(), + filter + ); +} diff --git a/src/filters/traits.rs b/src/filters/traits.rs new file mode 100644 index 00000000..61b7e8fe --- /dev/null +++ b/src/filters/traits.rs @@ -0,0 +1,27 @@ +use super::*; + +// references: +// https://dev.to/magnusstrale/rust-trait-objects-in-a-vector-non-trivial-4co5 +// https://stackoverflow.com/questions/25339603/how-to-test-for-equality-between-trait-objects + +/// FeroxFilter trait; represents different types of possible filters that can be applied to +/// responses +pub trait FeroxFilter: Debug + Send + Sync { + /// Determine whether or not this particular filter should be applied or not + fn should_filter_response(&self, response: &FeroxResponse) -> bool; + + /// delegates to the FeroxFilter-implementing type which gives us the actual type of self + fn box_eq(&self, other: &dyn Any) -> bool; + + /// gives us `other` as Any in box_eq + fn as_any(&self) -> &dyn Any; +} + +/// implementation of PartialEq, necessary long-form due to "trait cannot be made into an object" +/// error when attempting to derive PartialEq on the trait itself +impl PartialEq for Box { + /// Perform a comparison of two implementors of the FeroxFilter trait + fn eq(&self, other: &Box) -> bool { + self.box_eq(other.as_any()) + } +} diff --git a/src/filters/wildcard.rs b/src/filters/wildcard.rs new file mode 100644 index 00000000..ed46df58 --- /dev/null +++ b/src/filters/wildcard.rs @@ -0,0 +1,73 @@ +use super::*; + +/// Data holder for two pieces of data needed when auto-filtering out wildcard responses +/// +/// `dynamic` is the size of the response that will later be combined with the length +/// of the path of the url requested and used to determine interesting pages from custom +/// 404s where the requested url is reflected back in the response +/// +/// `size` is size of the response that should be included with filters passed via runtime +/// configuration and any static wildcard lengths. +#[derive(Debug, Default, Clone, PartialEq)] +pub struct WildcardFilter { + /// size of the response that will later be combined with the length of the path of the url + /// requested + pub dynamic: u64, + + /// size of the response that should be included with filters passed via runtime configuration + pub size: u64, +} + +/// implementation of FeroxFilter for WildcardFilter +impl FeroxFilter for WildcardFilter { + /// Examine size, dynamic, and content_len to determine whether or not the response received + /// is a wildcard response and therefore should be filtered out + fn should_filter_response(&self, response: &FeroxResponse) -> bool { + log::trace!("enter: should_filter_response({:?} {})", self, response); + + // quick return if dont_filter is set + if CONFIGURATION.dont_filter { + // --dont-filter applies specifically to wildcard filters, it is not a 100% catch all + // for not filtering anything. As such, it should live in the implementation of + // a wildcard filter + return false; + } + + if self.size > 0 && self.size == response.content_length() { + // static wildcard size found during testing + // size isn't default, size equals response length, and auto-filter is on + log::debug!("static wildcard: filtered out {}", response.url()); + log::trace!("exit: should_filter_response -> true"); + return true; + } + + if self.dynamic > 0 { + // dynamic wildcard offset found during testing + + // I'm about to manually split this url path instead of using reqwest::Url's + // builtin parsing. The reason is that they call .split() on the url path + // except that I don't want an empty string taking up the last index in the + // event that the url ends with a forward slash. It's ugly enough to be split + // into its own function for readability. + let url_len = get_url_path_length(&response.url()); + + if url_len + self.dynamic == response.content_length() { + log::debug!("dynamic wildcard: filtered out {}", response.url()); + log::trace!("exit: should_filter_response -> true"); + return true; + } + } + log::trace!("exit: should_filter_response -> false"); + false + } + + /// Compare one WildcardFilter to another + fn box_eq(&self, other: &dyn Any) -> bool { + other.downcast_ref::().map_or(false, |a| self == a) + } + + /// Return self as Any for dynamic dispatch purposes + fn as_any(&self) -> &dyn Any { + self + } +} diff --git a/src/filters/words.rs b/src/filters/words.rs new file mode 100644 index 00000000..59c58858 --- /dev/null +++ b/src/filters/words.rs @@ -0,0 +1,33 @@ +use super::*; + +/// Simple implementor of FeroxFilter; used to filter out responses based on the number of words +/// in a Response body; specified using -W|--filter-words +#[derive(Default, Debug, PartialEq)] +pub struct WordsFilter { + /// Number of words in a Response's body that should be filtered + pub word_count: usize, +} + +/// implementation of FeroxFilter for WordsFilter +impl FeroxFilter for WordsFilter { + /// Check `word_count` against what was passed in via -W|--filter-words + fn should_filter_response(&self, response: &FeroxResponse) -> bool { + log::trace!("enter: should_filter_response({:?} {})", self, response); + + let result = response.word_count() == self.word_count; + + log::trace!("exit: should_filter_response -> {}", result); + + result + } + + /// Compare one WordsFilter to another + fn box_eq(&self, other: &dyn Any) -> bool { + other.downcast_ref::().map_or(false, |a| self == a) + } + + /// Return self as Any for dynamic dispatch purposes + fn as_any(&self) -> &dyn Any { + self + } +} diff --git a/src/main.rs b/src/main.rs index 2d5c73a9..e46d44d0 100644 --- a/src/main.rs +++ b/src/main.rs @@ -409,8 +409,6 @@ async fn clean_up( stats_handle, save_output ); - update_stat!(tx_stats, StatCommand::Exit); // send exit command and await the end of the future - drop(tx_term); log::trace!("dropped terminal output handler's transmitter"); @@ -442,6 +440,7 @@ async fn clean_up( log::trace!("done awaiting file output handler's receiver"); } + update_stat!(tx_stats, StatCommand::Exit); // send exit command and await the end of the future stats_handle.await.unwrap_or_default(); // mark all scans complete so the terminal input handler will exit cleanly diff --git a/src/scanner.rs b/src/scanner.rs index 1295eadd..44e8dd2f 100644 --- a/src/scanner.rs +++ b/src/scanner.rs @@ -22,7 +22,7 @@ use futures::{ use fuzzyhash::FuzzyHash; use lazy_static::lazy_static; use regex::Regex; -use reqwest::Url; +use reqwest::{StatusCode, Url}; #[cfg(not(test))] use std::process::exit; use std::{ @@ -260,8 +260,9 @@ fn response_is_directory(response: &FeroxResponse) -> bool { return false; } } - } else if response.status().is_success() { - // status code is 2xx, need to check if it ends in / + } else if response.status().is_success() || matches!(response.status(), &StatusCode::FORBIDDEN) + { + // status code is 2xx or 403, need to check if it ends in / if response.url().as_str().ends_with('/') { log::debug!("{} is directory suitable for recursion", response.url()); @@ -464,10 +465,14 @@ async fn make_requests( if !CONFIGURATION.no_recursion { log::debug!("Recursive extraction: {}", new_ferox_response); - if new_ferox_response.status().is_success() - && !new_ferox_response.url().as_str().ends_with('/') + if !new_ferox_response.url().as_str().ends_with('/') + && (new_ferox_response.status().is_success() + || matches!(new_ferox_response.status(), &StatusCode::FORBIDDEN)) { - // since all of these are 2xx, recursion is only attempted if the + // if the url doesn't end with a / + // and the response code is either a 2xx or 403 + + // since all of these are 2xx or 403, recursion is only attempted if the // url ends in a /. I am actually ok with adding the slash and not // adding it, as both have merit. Leaving it in for now to see how // things turn out (current as of: v1.1.0) @@ -540,8 +545,12 @@ async fn scan_robots_txt( } else if !CONFIGURATION.no_recursion { log::debug!("Directory extracted from robots.txt: {}", ferox_response); // todo this code is essentially the same as another piece around ~467 of this file - if ferox_response.status().is_success() && !ferox_response.url().as_str().ends_with('/') + if !ferox_response.url().as_str().ends_with('/') + && (ferox_response.status().is_success() + || matches!(ferox_response.status(), &StatusCode::FORBIDDEN)) { + // if the url doesn't end with a / + // and the response code is either a 2xx or 403 ferox_response.set_url(&format!("{}/", ferox_response.url())); } diff --git a/tests/test_extractor.rs b/tests/test_extractor.rs index b02667df..4d872d67 100644 --- a/tests/test_extractor.rs +++ b/tests/test_extractor.rs @@ -295,3 +295,54 @@ fn extractor_finds_robots_txt_links_and_displays_files_or_scans_directories() { assert_eq!(mock_scanned_file.hits(), 1); teardown_tmp_directory(tmp_dir); } + +#[test] +/// send a request to a page that contains a link that contains a directory that returns a 403 +/// --extract-links should find the link and make recurse into the 403 directory, finding LICENSE +fn extractor_recurses_into_403_directories() -> Result<(), Box> { + let srv = MockServer::start(); + let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?; + + let mock = srv.mock(|when, then| { + when.method(GET).path("/LICENSE"); + then.status(200) + .body(&srv.url("'/homepage/assets/img/icons/handshake.svg'")); + }); + + let mock_two = srv.mock(|when, then| { + when.method(GET).path("/homepage/assets/img/icons/LICENSE"); + then.status(200).body("that's just like, your opinion man"); + }); + + let forbidden_dir = srv.mock(|when, then| { + when.method(GET).path("/homepage/assets/img/icons/"); + then.status(403); + }); + + let cmd = Command::cargo_bin("feroxbuster") + .unwrap() + .arg("--url") + .arg(srv.url("/")) + .arg("--wordlist") + .arg(file.as_os_str()) + .arg("--extract-links") + .arg("--depth") // need to go past default 4 directories + .arg("0") + .unwrap(); + + cmd.assert().success().stdout( + predicate::str::contains("/LICENSE") + .count(2) + .and(predicate::str::contains("1w")) // link in /LICENSE + .and(predicate::str::contains("34c")) // recursed LICENSE + .and(predicate::str::contains( + "/homepage/assets/img/icons/LICENSE", + )), + ); + + assert_eq!(mock.hits(), 1); + assert_eq!(mock_two.hits(), 1); + assert_eq!(forbidden_dir.hits(), 1); + teardown_tmp_directory(tmp_dir); + Ok(()) +} diff --git a/tests/test_filters.rs b/tests/test_filters.rs index 293a6d58..235e2683 100644 --- a/tests/test_filters.rs +++ b/tests/test_filters.rs @@ -193,7 +193,7 @@ fn filters_size_should_filter_response() { #[test] /// create a FeroxResponse that should elicit a true from /// SimilarityFilter::should_filter_response -fn filter_similar_should_filter_response() { +fn filters_similar_should_filter_response() { let srv = MockServer::start(); let (tmp_dir, file) = setup_tmp_directory( &["not-similar".to_string(), "similar".to_string()], diff --git a/tests/test_scanner.rs b/tests/test_scanner.rs index 6fd77238..6dc3c551 100644 --- a/tests/test_scanner.rs +++ b/tests/test_scanner.rs @@ -546,3 +546,53 @@ fn scanner_single_request_scan_with_regex_filtered_result() { assert_eq!(filtered_mock.hits(), 1); teardown_tmp_directory(tmp_dir); } + +#[test] +/// send a request to a 403 directory, expect recursion to work into the 403 +fn scanner_recursion_works_with_403_directories() { + let srv = MockServer::start(); + let (tmp_dir, file) = + setup_tmp_directory(&["LICENSE".to_string(), "ignored/".to_string()], "wordlist").unwrap(); + + let mock = srv.mock(|when, then| { + when.method(GET).path("/LICENSE"); + then.status(200).body("this is a test"); + }); + + let forbidden_dir = srv.mock(|when, then| { + when.method(GET).path("/ignored/"); + then.status(403); + }); + + let found_anyway = srv.mock(|when, then| { + when.method(GET).path("/ignored/LICENSE"); + then.status(200) + .body("this is a test\nThat rug really tied the room together"); + }); + + let cmd = Command::cargo_bin("feroxbuster") + .unwrap() + .arg("--url") + .arg(srv.url("/")) + .arg("--wordlist") + .arg(file.as_os_str()) + .unwrap(); + + cmd.assert().success().stdout( + predicate::str::contains("/LICENSE") + .count(2) + .and(predicate::str::contains("200").count(2)) + .and(predicate::str::contains("403")) + .and(predicate::str::contains("53c")) + .and(predicate::str::contains("14c")) + .and(predicate::str::contains("0c")) + .and(predicate::str::contains("ignored").count(2)) + .and(predicate::str::contains("/ignored/LICENSE")), + ); + + assert_eq!(mock.hits(), 1); + assert_eq!(found_anyway.hits(), 1); + assert_eq!(forbidden_dir.hits(), 1); + + teardown_tmp_directory(tmp_dir); +}