907 dont skip dir listings (#1192)

* bumped version to 2.11.0

* updated deps

* new cli options

* added --request-file, --protocol, --scan-dir-listings

* added tests / clippy

* removed errant module definition

* implemented visible bar limiter

* many fixes; feature implemented i believe

* added banner test for limit-bars

* beginning troubleshooting of recursion panic

* put a bandaid on trace-level logging bug

* clippy

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "feroxbuster"
-version = "2.10.4"
+version = "2.11.0"
 authors = ["Ben 'epi' Risher (@epi052)"]
 license = "MIT"
 edition = "2021"
@@ -25,13 +25,13 @@ maintenance = { status = "actively-developed" }
 clap = { version = "4.5", features = ["wrap_help", "cargo"] }
 clap_complete = "4.5"
 regex = "1.10"
-lazy_static = "1.4"
+lazy_static = "1.5"
 dirs = "5.0"
 
 [dependencies]
 scraper = "0.19"
 futures = "0.3"
-tokio = { version = "1.38", features = ["full"] }
+tokio = { version = "1.39", features = ["full"] }
 tokio-util = { version = "0.7", features = ["codec"] }
 log = "0.4"
 env_logger = "0.11"
@@ -40,13 +40,11 @@ reqwest = { version = "0.12", features = ["socks", "native-tls-alpn"] }
 url = { version = "2.5", features = ["serde"] }
 serde_regex = "1.1"
 clap = { version = "4.5", features = ["wrap_help", "cargo"] }
-lazy_static = "1.4"
+lazy_static = "1.5"
 toml = "0.8"
 serde = { version = "1.0", features = ["derive", "rc"] }
 serde_json = "1.0"
-uuid = { version = "1.8", features = ["v4"] }
-# last known working version of indicatif; 0.17.5 has a bug that causes the 
-# scan menu to fail spectacularly 
+uuid = { version = "1.10", features = ["v4"] }
 indicatif = { version = "0.17.8" }
 console = "0.15"
 openssl = { version = "0.10", features = ["vendored"] }
@@ -69,7 +67,7 @@ self_update = { version = "0.40", features = [
 ] }
 
 [dev-dependencies]
-tempfile = "3.10"
+tempfile = "3.12"
 httpmock = "0.7"
 assert_cmd = "2.0"
 predicates = "3.1"

Makefile.toml

@@ -14,7 +14,7 @@ rm ferox-*.state
 # dependency management
 [tasks.upgrade-deps]
 command = "cargo"
-args = ["upgrade", "--exclude", "indicatif, self_update"]
+args = ["upgrade", "--exclude", "self_update"]
 
 [tasks.update]
 command = "cargo"

ferox-config.toml.example

@@ -45,6 +45,7 @@
 # dont_filter = true
 # extract_links = true
 # depth = 1
+# limit_bars = 3
 # force_recursion = true
 # filter_size = [5174]
 # filter_regex = ["^ignore me$"]
@@ -57,6 +58,9 @@
 # server_certs = ["/some/cert.pem", "/some/other/cert.pem"]
 # client_cert = "/some/client/cert.pem"
 # client_key = "/some/client/key.pem"
+# request_file = "/some/raw/request/file"
+# protocol = "http"
+# scan_dir_listings = true
 
 # headers can be specified on multiple lines or as an inline table
 #

shell_completions/_feroxbuster

@@ -15,17 +15,18 @@ _feroxbuster() {
 
     local context curcontext="$curcontext" state line
     _arguments "${_arguments_options[@]}" : \
-'-u+[The target URL (required, unless \[--stdin || --resume-from\] used)]:URL:_urls' \
-'--url=[The target URL (required, unless \[--stdin || --resume-from\] used)]:URL:_urls' \
+'-u+[The target URL (required, unless \[--stdin || --resume-from || --request-file\] used)]:URL:_urls' \
+'--url=[The target URL (required, unless \[--stdin || --resume-from || --request-file\] used)]:URL:_urls' \
 '(-u --url)--resume-from=[State file from which to resume a partially complete scan (ex. --resume-from ferox-1606586780.state)]:STATE_FILE:_files' \
+'(-u --url)--request-file=[Raw HTTP request file to use as a template for all requests]:REQUEST_FILE:_files' \
 '-p+[Proxy to use for requests (ex\: http(s)\://host\:port, socks5(h)\://host\:port)]:PROXY:_urls' \
 '--proxy=[Proxy to use for requests (ex\: http(s)\://host\:port, socks5(h)\://host\:port)]:PROXY:_urls' \
 '-P+[Send only unfiltered requests through a Replay Proxy, instead of all requests]:REPLAY_PROXY:_urls' \
 '--replay-proxy=[Send only unfiltered requests through a Replay Proxy, instead of all requests]:REPLAY_PROXY:_urls' \
 '*-R+[Status Codes to send through a Replay Proxy when found (default\: --status-codes value)]:REPLAY_CODE: ' \
 '*--replay-codes=[Status Codes to send through a Replay Proxy when found (default\: --status-codes value)]:REPLAY_CODE: ' \
-'-a+[Sets the User-Agent (default\: feroxbuster/2.10.4)]:USER_AGENT: ' \
-'--user-agent=[Sets the User-Agent (default\: feroxbuster/2.10.4)]:USER_AGENT: ' \
+'-a+[Sets the User-Agent (default\: feroxbuster/2.11.0)]:USER_AGENT: ' \
+'--user-agent=[Sets the User-Agent (default\: feroxbuster/2.11.0)]:USER_AGENT: ' \
 '*-x+[File extension(s) to search for (ex\: -x php -x pdf js); reads values (newline-separated) from file if input starts with an @ (ex\: @ext.txt)]:FILE_EXTENSION: ' \
 '*--extensions=[File extension(s) to search for (ex\: -x php -x pdf js); reads values (newline-separated) from file if input starts with an @ (ex\: @ext.txt)]:FILE_EXTENSION: ' \
 '*-m+[Which HTTP request method(s) should be sent (default\: GET)]:HTTP_METHODS: ' \
@@ -37,6 +38,7 @@ _feroxbuster() {
 '*--cookies=[Specify HTTP cookies to be used in each request (ex\: -b stuff=things)]:COOKIE: ' \
 '*-Q+[Request'\''s URL query parameters (ex\: -Q token=stuff -Q secret=key)]:QUERY: ' \
 '*--query=[Request'\''s URL query parameters (ex\: -Q token=stuff -Q secret=key)]:QUERY: ' \
+'--protocol=[Specify the protocol to use when targeting via --request-file or --url with domain only (default\: https)]:PROTOCOL: ' \
 '*--dont-scan=[URL(s) or Regex Pattern(s) to exclude from recursion/scans]:URL: ' \
 '*-S+[Filter out messages of a particular size (ex\: -S 5120 -S 4927,1970)]:SIZE: ' \
 '*--filter-size=[Filter out messages of a particular size (ex\: -S 5120 -S 4927,1970)]:SIZE: ' \
@@ -74,11 +76,12 @@ _feroxbuster() {
 '-o+[Output file to write results to (use w/ --json for JSON entries)]:FILE:_files' \
 '--output=[Output file to write results to (use w/ --json for JSON entries)]:FILE:_files' \
 '--debug-log=[Output file to write log entries (use w/ --json for JSON entries)]:FILE:_files' \
+'--limit-bars=[Number of directory scan bars to show at any given time (default\: no limit)]:NUM_BARS_TO_SHOW: ' \
 '(-u --url)--stdin[Read url(s) from STDIN]' \
 '(-p --proxy -k --insecure --burp-replay)--burp[Set --proxy to http\://127.0.0.1\:8080 and set --insecure to true]' \
 '(-P --replay-proxy -k --insecure)--burp-replay[Set --replay-proxy to http\://127.0.0.1\:8080 and set --insecure to true]' \
 '(--rate-limit --auto-bail)--smart[Set --auto-tune, --collect-words, and --collect-backups to true]' \
-'(--rate-limit --auto-bail)--thorough[Use the same settings as --smart and set --collect-extensions to true]' \
+'(--rate-limit --auto-bail)--thorough[Use the same settings as --smart and set --collect-extensions and --scan-dir-listings to true]' \
 '-A[Use a random User-Agent]' \
 '--random-agent[Use a random User-Agent]' \
 '-f[Append / to each request'\''s URL]' \
@@ -101,6 +104,7 @@ _feroxbuster() {
 '--collect-extensions[Automatically discover extensions and add them to --extensions (unless they'\''re in --dont-collect)]' \
 '-g[Automatically discover important words from within responses and add them to the wordlist]' \
 '--collect-words[Automatically discover important words from within responses and add them to the wordlist]' \
+'--scan-dir-listings[Force scans to recurse into directory listings]' \
 '(--silent)*-v[Increase verbosity level (use -vv or more for greater effect. \[CAUTION\] 4 -v'\''s is probably too much)]' \
 '(--silent)*--verbosity[Increase verbosity level (use -vv or more for greater effect. \[CAUTION\] 4 -v'\''s is probably too much)]' \
 '(-q --quiet)--silent[Only print URLs (or JSON w/ --json) + turn off logging (good for piping a list of urls to other commands)]' \