Skip to content

Commit

Permalink
Merge pull request #834 from epi052/827-load-wordlist-from-url
Browse files Browse the repository at this point in the history 
load wordlist from url; change some defaults/fix some bugs
  • Loading branch information
@epi052
epi052 authored Mar 18, 2023
2 parents b786578 + 46d0050 commit 5d96658
Show file tree
Hide file tree
Showing 13 changed files with 253 additions and 145 deletions.
190 changes: 105 additions & 85 deletions Cargo.lock
View file

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions Cargo.toml
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[package]
name = "feroxbuster"
version = "2.9.1"
version = "2.9.2"
authors = ["Ben 'epi' Risher (@epi052)"]
license = "MIT"
edition = "2021"
Expand Down Expand Up @@ -62,7 +62,7 @@ self_update = {version = "0.36.0", features = ["archive-tar", "compression-flate
tempfile = "3.3.0"
httpmock = "0.6.6"
assert_cmd = "2.0.4"
predicates = "2.1.1"
predicates = "3.0.1"

[profile.release]
lto = true
Expand Down
15 changes: 8 additions & 7 deletions shell_completions/_feroxbuster
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,8 @@ _feroxbuster() {
'--replay-proxy=[Send only unfiltered requests through a Replay Proxy, instead of all requests]:REPLAY_PROXY:_urls' \
'*-R+[Status Codes to send through a Replay Proxy when found (default: --status-codes value)]:REPLAY_CODE: ' \
'*--replay-codes=[Status Codes to send through a Replay Proxy when found (default: --status-codes value)]:REPLAY_CODE: ' \
'-a+[Sets the User-Agent (default: feroxbuster/2.9.1)]:USER_AGENT: ' \
'--user-agent=[Sets the User-Agent (default: feroxbuster/2.9.1)]:USER_AGENT: ' \
'-a+[Sets the User-Agent (default: feroxbuster/2.9.2)]:USER_AGENT: ' \
'--user-agent=[Sets the User-Agent (default: feroxbuster/2.9.2)]:USER_AGENT: ' \
'*-x+[File extension(s) to search for (ex: -x php -x pdf js)]:FILE_EXTENSION: ' \
'*--extensions=[File extension(s) to search for (ex: -x php -x pdf js)]:FILE_EXTENSION: ' \
'*-m+[Which HTTP request method(s) should be sent (default: GET)]:HTTP_METHODS: ' \
Expand Down Expand Up @@ -62,8 +62,8 @@ _feroxbuster() {
'--parallel=[Run parallel feroxbuster instances (one child process per url passed via stdin)]:PARALLEL_SCANS: ' \
'(--auto-tune)--rate-limit=[Limit number of requests per second (per directory) (default: 0, i.e. no limit)]:RATE_LIMIT: ' \
'--time-limit=[Limit total run time of all scans (ex: --time-limit 10m)]:TIME_SPEC: ' \
'-w+[Path to the wordlist]:FILE:_files' \
'--wordlist=[Path to the wordlist]:FILE:_files' \
'-w+[Path or URL of the wordlist]:FILE:_files' \
'--wordlist=[Path or URL of the wordlist]:FILE:_files' \
'*-I+[File extension(s) to Ignore while collecting extensions (only used with --collect-extensions)]:FILE_EXTENSION: ' \
'*--dont-collect=[File extension(s) to Ignore while collecting extensions (only used with --collect-extensions)]:FILE_EXTENSION: ' \
'-o+[Output file to write results to (use w/ --json for JSON entries)]:FILE:_files' \
Expand All @@ -72,7 +72,7 @@ _feroxbuster() {
'(-u --url)--stdin[Read url(s) from STDIN]' \
'(-p --proxy -k --insecure --burp-replay)--burp[Set --proxy to http://127.0.0.1:8080 and set --insecure to true]' \
'(-P --replay-proxy -k --insecure)--burp-replay[Set --replay-proxy to http://127.0.0.1:8080 and set --insecure to true]' \
'(--rate-limit --auto-bail)--smart[Set --extract-links, --auto-tune, --collect-words, and --collect-backups to true]' \
'(--rate-limit --auto-bail)--smart[Set --auto-tune, --collect-words, and --collect-backups to true]' \
'(--rate-limit --auto-bail)--thorough[Use the same settings as --smart and set --collect-extensions to true]' \
'-A[Use a random User-Agent]' \
'--random-agent[Use a random User-Agent]' \
Expand All @@ -85,8 +85,9 @@ _feroxbuster() {
'-n[Do not scan recursively]' \
'--no-recursion[Do not scan recursively]' \
'(-n --no-recursion)--force-recursion[Force recursion attempts on all '\''found'\'' endpoints (still respects recursion depth)]' \
'-e[Extract links from response body (html, javascript, etc...); make new requests based on findings]' \
'--extract-links[Extract links from response body (html, javascript, etc...); make new requests based on findings]' \
'-e[Extract links from response body (html, javascript, etc...); make new requests based on findings (default: true)]' \
'--extract-links[Extract links from response body (html, javascript, etc...); make new requests based on findings (default: true)]' \
'--dont-extract-links[Don'\''t extract links from response body (html, javascript, etc...)]' \
'(--auto-bail)--auto-tune[Automatically lower scan rate when an excessive amount of errors are encountered]' \
'--auto-bail[Automatically stop scanning when an excessive amount of errors are encountered]' \
'-D[Don'\''t auto-filter wildcard responses]' \
Expand Down
15 changes: 8 additions & 7 deletions shell_completions/_feroxbuster.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,8 @@ Register-ArgumentCompleter -Native -CommandName 'feroxbuster' -ScriptBlock {
[CompletionResult]::new('--replay-proxy', 'replay-proxy', [CompletionResultType]::ParameterName, 'Send only unfiltered requests through a Replay Proxy, instead of all requests')
[CompletionResult]::new('-R', 'R', [CompletionResultType]::ParameterName, 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)')
[CompletionResult]::new('--replay-codes', 'replay-codes', [CompletionResultType]::ParameterName, 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)')
[CompletionResult]::new('-a', 'a', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/2.9.1)')
[CompletionResult]::new('--user-agent', 'user-agent', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/2.9.1)')
[CompletionResult]::new('-a', 'a', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/2.9.2)')
[CompletionResult]::new('--user-agent', 'user-agent', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/2.9.2)')
[CompletionResult]::new('-x', 'x', [CompletionResultType]::ParameterName, 'File extension(s) to search for (ex: -x php -x pdf js)')
[CompletionResult]::new('--extensions', 'extensions', [CompletionResultType]::ParameterName, 'File extension(s) to search for (ex: -x php -x pdf js)')
[CompletionResult]::new('-m', 'm', [CompletionResultType]::ParameterName, 'Which HTTP request method(s) should be sent (default: GET)')
Expand Down Expand Up @@ -68,8 +68,8 @@ Register-ArgumentCompleter -Native -CommandName 'feroxbuster' -ScriptBlock {
[CompletionResult]::new('--parallel', 'parallel', [CompletionResultType]::ParameterName, 'Run parallel feroxbuster instances (one child process per url passed via stdin)')
[CompletionResult]::new('--rate-limit', 'rate-limit', [CompletionResultType]::ParameterName, 'Limit number of requests per second (per directory) (default: 0, i.e. no limit)')
[CompletionResult]::new('--time-limit', 'time-limit', [CompletionResultType]::ParameterName, 'Limit total run time of all scans (ex: --time-limit 10m)')
[CompletionResult]::new('-w', 'w', [CompletionResultType]::ParameterName, 'Path to the wordlist')
[CompletionResult]::new('--wordlist', 'wordlist', [CompletionResultType]::ParameterName, 'Path to the wordlist')
[CompletionResult]::new('-w', 'w', [CompletionResultType]::ParameterName, 'Path or URL of the wordlist')
[CompletionResult]::new('--wordlist', 'wordlist', [CompletionResultType]::ParameterName, 'Path or URL of the wordlist')
[CompletionResult]::new('-I', 'I', [CompletionResultType]::ParameterName, 'File extension(s) to Ignore while collecting extensions (only used with --collect-extensions)')
[CompletionResult]::new('--dont-collect', 'dont-collect', [CompletionResultType]::ParameterName, 'File extension(s) to Ignore while collecting extensions (only used with --collect-extensions)')
[CompletionResult]::new('-o', 'o', [CompletionResultType]::ParameterName, 'Output file to write results to (use w/ --json for JSON entries)')
Expand All @@ -78,7 +78,7 @@ Register-ArgumentCompleter -Native -CommandName 'feroxbuster' -ScriptBlock {
[CompletionResult]::new('--stdin', 'stdin', [CompletionResultType]::ParameterName, 'Read url(s) from STDIN')
[CompletionResult]::new('--burp', 'burp', [CompletionResultType]::ParameterName, 'Set --proxy to http://127.0.0.1:8080 and set --insecure to true')
[CompletionResult]::new('--burp-replay', 'burp-replay', [CompletionResultType]::ParameterName, 'Set --replay-proxy to http://127.0.0.1:8080 and set --insecure to true')
[CompletionResult]::new('--smart', 'smart', [CompletionResultType]::ParameterName, 'Set --extract-links, --auto-tune, --collect-words, and --collect-backups to true')
[CompletionResult]::new('--smart', 'smart', [CompletionResultType]::ParameterName, 'Set --auto-tune, --collect-words, and --collect-backups to true')
[CompletionResult]::new('--thorough', 'thorough', [CompletionResultType]::ParameterName, 'Use the same settings as --smart and set --collect-extensions to true')
[CompletionResult]::new('-A', 'A', [CompletionResultType]::ParameterName, 'Use a random User-Agent')
[CompletionResult]::new('--random-agent', 'random-agent', [CompletionResultType]::ParameterName, 'Use a random User-Agent')
Expand All @@ -91,8 +91,9 @@ Register-ArgumentCompleter -Native -CommandName 'feroxbuster' -ScriptBlock {
[CompletionResult]::new('-n', 'n', [CompletionResultType]::ParameterName, 'Do not scan recursively')
[CompletionResult]::new('--no-recursion', 'no-recursion', [CompletionResultType]::ParameterName, 'Do not scan recursively')
[CompletionResult]::new('--force-recursion', 'force-recursion', [CompletionResultType]::ParameterName, 'Force recursion attempts on all ''found'' endpoints (still respects recursion depth)')
[CompletionResult]::new('-e', 'e', [CompletionResultType]::ParameterName, 'Extract links from response body (html, javascript, etc...); make new requests based on findings')
[CompletionResult]::new('--extract-links', 'extract-links', [CompletionResultType]::ParameterName, 'Extract links from response body (html, javascript, etc...); make new requests based on findings')
[CompletionResult]::new('-e', 'e', [CompletionResultType]::ParameterName, 'Extract links from response body (html, javascript, etc...); make new requests based on findings (default: true)')
[CompletionResult]::new('--extract-links', 'extract-links', [CompletionResultType]::ParameterName, 'Extract links from response body (html, javascript, etc...); make new requests based on findings (default: true)')
[CompletionResult]::new('--dont-extract-links', 'dont-extract-links', [CompletionResultType]::ParameterName, 'Don''t extract links from response body (html, javascript, etc...)')
[CompletionResult]::new('--auto-tune', 'auto-tune', [CompletionResultType]::ParameterName, 'Automatically lower scan rate when an excessive amount of errors are encountered')
[CompletionResult]::new('--auto-bail', 'auto-bail', [CompletionResultType]::ParameterName, 'Automatically stop scanning when an excessive amount of errors are encountered')
[CompletionResult]::new('-D', 'D', [CompletionResultType]::ParameterName, 'Don''t auto-filter wildcard responses')
Expand Down
2 changes: 1 addition & 1 deletion shell_completions/feroxbuster.bash
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ _feroxbuster() {

case "${cmd}" in
feroxbuster)
opts="-u -p -P -R -a -A -x -m -H -b -Q -f -S -X -W -N -C -s -T -r -k -t -n -d -e -L -w -D -E -B -g -I -v -q -o -U -h -V --url --stdin --resume-from --burp --burp-replay --smart --thorough --proxy --replay-proxy --replay-codes --user-agent --random-agent --extensions --methods --data --headers --cookies --query --add-slash --dont-scan --filter-size --filter-regex --filter-words --filter-lines --filter-status --filter-similar-to --status-codes --timeout --redirects --insecure --threads --no-recursion --depth --force-recursion --extract-links --scan-limit --parallel --rate-limit --time-limit --wordlist --auto-tune --auto-bail --dont-filter --collect-extensions --collect-backups --collect-words --dont-collect --verbosity --silent --quiet --json --output --debug-log --no-state --update --help --version"
opts="-u -p -P -R -a -A -x -m -H -b -Q -f -S -X -W -N -C -s -T -r -k -t -n -d -e -L -w -D -E -B -g -I -v -q -o -U -h -V --url --stdin --resume-from --burp --burp-replay --smart --thorough --proxy --replay-proxy --replay-codes --user-agent --random-agent --extensions --methods --data --headers --cookies --query --add-slash --dont-scan --filter-size --filter-regex --filter-words --filter-lines --filter-status --filter-similar-to --status-codes --timeout --redirects --insecure --threads --no-recursion --depth --force-recursion --extract-links --dont-extract-links --scan-limit --parallel --rate-limit --time-limit --wordlist --auto-tune --auto-bail --dont-filter --collect-extensions --collect-backups --collect-words --dont-collect --verbosity --silent --quiet --json --output --debug-log --no-state --update --help --version"
if [[ ${cur} == -* || ${COMP_CWORD} -eq 1 ]] ; then
COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
return 0
Expand Down
15 changes: 8 additions & 7 deletions shell_completions/feroxbuster.elv
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,8 @@ set edit:completion:arg-completer[feroxbuster] = {|@words|
cand --replay-proxy 'Send only unfiltered requests through a Replay Proxy, instead of all requests'
cand -R 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)'
cand --replay-codes 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)'
cand -a 'Sets the User-Agent (default: feroxbuster/2.9.1)'
cand --user-agent 'Sets the User-Agent (default: feroxbuster/2.9.1)'
cand -a 'Sets the User-Agent (default: feroxbuster/2.9.2)'
cand --user-agent 'Sets the User-Agent (default: feroxbuster/2.9.2)'
cand -x 'File extension(s) to search for (ex: -x php -x pdf js)'
cand --extensions 'File extension(s) to search for (ex: -x php -x pdf js)'
cand -m 'Which HTTP request method(s) should be sent (default: GET)'
Expand Down Expand Up @@ -65,8 +65,8 @@ set edit:completion:arg-completer[feroxbuster] = {|@words|
cand --parallel 'Run parallel feroxbuster instances (one child process per url passed via stdin)'
cand --rate-limit 'Limit number of requests per second (per directory) (default: 0, i.e. no limit)'
cand --time-limit 'Limit total run time of all scans (ex: --time-limit 10m)'
cand -w 'Path to the wordlist'
cand --wordlist 'Path to the wordlist'
cand -w 'Path or URL of the wordlist'
cand --wordlist 'Path or URL of the wordlist'
cand -I 'File extension(s) to Ignore while collecting extensions (only used with --collect-extensions)'
cand --dont-collect 'File extension(s) to Ignore while collecting extensions (only used with --collect-extensions)'
cand -o 'Output file to write results to (use w/ --json for JSON entries)'
Expand All @@ -75,7 +75,7 @@ set edit:completion:arg-completer[feroxbuster] = {|@words|
cand --stdin 'Read url(s) from STDIN'
cand --burp 'Set --proxy to http://127.0.0.1:8080 and set --insecure to true'
cand --burp-replay 'Set --replay-proxy to http://127.0.0.1:8080 and set --insecure to true'
cand --smart 'Set --extract-links, --auto-tune, --collect-words, and --collect-backups to true'
cand --smart 'Set --auto-tune, --collect-words, and --collect-backups to true'
cand --thorough 'Use the same settings as --smart and set --collect-extensions to true'
cand -A 'Use a random User-Agent'
cand --random-agent 'Use a random User-Agent'
Expand All @@ -88,8 +88,9 @@ set edit:completion:arg-completer[feroxbuster] = {|@words|
cand -n 'Do not scan recursively'
cand --no-recursion 'Do not scan recursively'
cand --force-recursion 'Force recursion attempts on all ''found'' endpoints (still respects recursion depth)'
cand -e 'Extract links from response body (html, javascript, etc...); make new requests based on findings'
cand --extract-links 'Extract links from response body (html, javascript, etc...); make new requests based on findings'
cand -e 'Extract links from response body (html, javascript, etc...); make new requests based on findings (default: true)'
cand --extract-links 'Extract links from response body (html, javascript, etc...); make new requests based on findings (default: true)'
cand --dont-extract-links 'Don''t extract links from response body (html, javascript, etc...)'
cand --auto-tune 'Automatically lower scan rate when an excessive amount of errors are encountered'
cand --auto-bail 'Automatically stop scanning when an excessive amount of errors are encountered'
cand -D 'Don''t auto-filter wildcard responses'
Expand Down
21 changes: 10 additions & 11 deletions src/config/container.rs
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
use super::utils::{
depth, ignored_extensions, methods, report_and_exit, save_state, serialized_type, status_codes,
threads, timeout, user_agent, wordlist, OutputLevel, RequesterPolicy,
depth, extract_links, ignored_extensions, methods, report_and_exit, save_state,
serialized_type, status_codes, threads, timeout, user_agent, wordlist, OutputLevel,
RequesterPolicy,
};
use crate::config::determine_output_level;
use crate::config::utils::determine_requester_policy;
Expand Down Expand Up @@ -214,7 +215,7 @@ pub struct Configuration {
pub no_recursion: bool,

/// Extract links from html/javscript
#[serde(default)]
#[serde(default = "extract_links")]
pub extract_links: bool,

/// Append / to each request
Expand Down Expand Up @@ -328,6 +329,7 @@ impl Default for Configuration {
let kind = serialized_type();
let output_level = OutputLevel::Default;
let requester_policy = RequesterPolicy::Default;
let extract_links = extract_links();

Configuration {
kind,
Expand All @@ -336,6 +338,7 @@ impl Default for Configuration {
user_agent,
replay_codes,
status_codes,
extract_links,
replay_client,
requester_policy,
dont_filter: false,
Expand All @@ -355,7 +358,6 @@ impl Default for Configuration {
insecure: false,
redirects: false,
no_recursion: false,
extract_links: false,
random_agent: false,
collect_extensions: false,
collect_backups: false,
Expand Down Expand Up @@ -398,7 +400,7 @@ impl Configuration {
///
/// - **timeout**: `5` seconds
/// - **redirects**: `false`
/// - **extract-links**: `false`
/// - **extract_links**: `true`
/// - **wordlist**: [`DEFAULT_WORDLIST`](constant.DEFAULT_WORDLIST.html)
/// - **config**: `None`
/// - **threads**: `50`
Expand Down Expand Up @@ -807,11 +809,8 @@ impl Configuration {
config.add_slash = true;
}

if came_from_cli!(args, "extract_links")
|| came_from_cli!(args, "smart")
|| came_from_cli!(args, "thorough")
{
config.extract_links = true;
if came_from_cli!(args, "dont_extract_links") {
config.extract_links = false;
}

if came_from_cli!(args, "json") {
Expand Down Expand Up @@ -997,7 +996,7 @@ impl Configuration {
update_if_not_default!(&mut conf.redirects, new.redirects, false);
update_if_not_default!(&mut conf.insecure, new.insecure, false);
update_if_not_default!(&mut conf.force_recursion, new.force_recursion, false);
update_if_not_default!(&mut conf.extract_links, new.extract_links, false);
update_if_not_default!(&mut conf.extract_links, new.extract_links, extract_links());
update_if_not_default!(&mut conf.extensions, new.extensions, Vec::<String>::new());
update_if_not_default!(&mut conf.methods, new.methods, methods());
update_if_not_default!(&mut conf.data, new.data, Vec::<u8>::new());
Expand Down
Loading

0 comments on commit 5d96658

Please sign in to comment.