From ece0161d1bc4b3e3ad7a01a72021d7f0ed72889d Mon Sep 17 00:00:00 2001 From: Nikita Bulai Date: Mon, 11 May 2020 18:34:45 +0300 Subject: [PATCH] Add new providers --- CHANGELOG.md | 3 +- Gemfile | 2 +- lib/proxy_fetcher.rb | 2 + .../providers/free_proxy_list.rb | 21 -------- lib/proxy_fetcher/providers/mtpro.rb | 43 +++++++++++++++++ lib/proxy_fetcher/providers/proxypedia.rb | 48 +++++++++++++++++++ spec/proxy_fetcher/providers/mtpro_spec.rb | 11 +++++ .../providers/proxypedia_spec.rb | 11 +++++ 8 files changed, 118 insertions(+), 23 deletions(-) create mode 100644 lib/proxy_fetcher/providers/mtpro.rb create mode 100644 lib/proxy_fetcher/providers/proxypedia.rb create mode 100644 spec/proxy_fetcher/providers/mtpro_spec.rb create mode 100644 spec/proxy_fetcher/providers/proxypedia_spec.rb diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a5424d..fceb843 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,8 @@ Reverse Chronological Order: ## `master` -* Add your description here +* Add MTPro provider +* Add Proxypedia provider ## `0.13.0` (2020-03-09) diff --git a/Gemfile b/Gemfile index 68d5291..23c98dd 100644 --- a/Gemfile +++ b/Gemfile @@ -6,7 +6,7 @@ gemspec gem "nokogiri", "~> 1.8" gem "oga", "~> 3.2" -gem "rubocop", "~> 0.74" +gem "rubocop", "~> 0.80" group :test do gem "coveralls", require: false diff --git a/lib/proxy_fetcher.rb b/lib/proxy_fetcher.rb index 3f722b9..456428c 100644 --- a/lib/proxy_fetcher.rb +++ b/lib/proxy_fetcher.rb @@ -33,10 +33,12 @@ module ProxyFetcher # ProxyFetcher providers namespace module Providers require File.dirname(__FILE__) + "/proxy_fetcher/providers/base" + require File.dirname(__FILE__) + "/proxy_fetcher/providers/proxypedia" require File.dirname(__FILE__) + "/proxy_fetcher/providers/free_proxy_list" require File.dirname(__FILE__) + "/proxy_fetcher/providers/free_proxy_list_ssl" require File.dirname(__FILE__) + "/proxy_fetcher/providers/gather_proxy" require File.dirname(__FILE__) + "/proxy_fetcher/providers/http_tunnel" + require File.dirname(__FILE__) + "/proxy_fetcher/providers/mtpro" require File.dirname(__FILE__) + "/proxy_fetcher/providers/proxy_list" require File.dirname(__FILE__) + "/proxy_fetcher/providers/xroxy" end diff --git a/lib/proxy_fetcher/providers/free_proxy_list.rb b/lib/proxy_fetcher/providers/free_proxy_list.rb index 0672b9c..418aef9 100644 --- a/lib/proxy_fetcher/providers/free_proxy_list.rb +++ b/lib/proxy_fetcher/providers/free_proxy_list.rb @@ -45,27 +45,6 @@ def to_proxy(html_node) # def parse_type(html_node) https = html_node.content_at("td[6]") - # frozen_string_literal: true - # FreeProxyList provider class. - # Provider URL to fetch proxy list - # [NOTE] Doesn't support filtering - # Converts HTML node (entry of N tags) to ProxyFetcher::Proxy - # object. - # - # @param html_node [Object] - # HTML node from the ProxyFetcher::Document DOM model. - # - # @return [ProxyFetcher::Proxy] - # Proxy object - # - # Parses HTML node to extract proxy type. - # - # @param html_node [Object] - # HTML node from the ProxyFetcher::Document DOM model. - # - # @return [String] - # Proxy type - # https&.casecmp("yes")&.zero? ? ProxyFetcher::Proxy::HTTPS : ProxyFetcher::Proxy::HTTP end end diff --git a/lib/proxy_fetcher/providers/mtpro.rb b/lib/proxy_fetcher/providers/mtpro.rb new file mode 100644 index 0000000..b3011ac --- /dev/null +++ b/lib/proxy_fetcher/providers/mtpro.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +require "json" + +module ProxyFetcher + module Providers + # GatherProxy provider class. + class MTPro < Base + # Provider URL to fetch proxy list + def provider_url + "https://mtpro.xyz/api/?type=socks" + end + + def load_proxy_list(filters = {}) + html = load_html(provider_url, filters) + JSON.parse(html) + rescue JSON::ParserError + [] + end + + # Converts HTML node (entry of N tags) to ProxyFetcher::Proxy + # object. + # + # @param node [Object] + # HTML node from the ProxyFetcher::Document DOM model. + # + # @return [ProxyFetcher::Proxy] + # Proxy object + # + def to_proxy(node) + ProxyFetcher::Proxy.new.tap do |proxy| + proxy.addr = node["ip"] + proxy.port = Integer(node["port"]) + proxy.country = node["country"] + proxy.anonymity = "Unknown" + proxy.type = ProxyFetcher::Proxy::SOCKS5 + end + end + end + + ProxyFetcher::Configuration.register_provider(:mtpro, MTPro) + end +end diff --git a/lib/proxy_fetcher/providers/proxypedia.rb b/lib/proxy_fetcher/providers/proxypedia.rb new file mode 100644 index 0000000..4816bb3 --- /dev/null +++ b/lib/proxy_fetcher/providers/proxypedia.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +module ProxyFetcher + module Providers + # FreeProxyList provider class. + class Proxypedia < Base + # Provider URL to fetch proxy list + def provider_url + "https://proxypedia.org" + end + + # [NOTE] Doesn't support filtering + def xpath + "//main/ul/li[position()>1]" + end + + # Converts HTML node (entry of N tags) to ProxyFetcher::Proxy + # object.] + # + # @param html_node [Object] + # HTML node from the ProxyFetcher::Document DOM model. + # + # @return [ProxyFetcher::Proxy] + # Proxy object + # + def to_proxy(html_node) + addr, port = html_node.content_at("a").to_s.split(":") + + ProxyFetcher::Proxy.new.tap do |proxy| + proxy.addr = addr + proxy.port = Integer(port) + proxy.country = parse_country(html_node) + proxy.anonymity = "Unknown" + proxy.type = ProxyFetcher::Proxy::HTTP + end + end + + private + + def parse_country(html_node) + text = html_node.content.to_s + text[/\((.+?)\)/, 1] || "Unknown" + end + end + + ProxyFetcher::Configuration.register_provider(:proxypedia, Proxypedia) + end +end diff --git a/spec/proxy_fetcher/providers/mtpro_spec.rb b/spec/proxy_fetcher/providers/mtpro_spec.rb new file mode 100644 index 0000000..2f0b5d5 --- /dev/null +++ b/spec/proxy_fetcher/providers/mtpro_spec.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +require "spec_helper" + +describe ProxyFetcher::Providers::MTPro do + before :all do + ProxyFetcher.config.provider = :mtpro + end + + it_behaves_like "a manager" +end diff --git a/spec/proxy_fetcher/providers/proxypedia_spec.rb b/spec/proxy_fetcher/providers/proxypedia_spec.rb new file mode 100644 index 0000000..8446414 --- /dev/null +++ b/spec/proxy_fetcher/providers/proxypedia_spec.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +require "spec_helper" + +describe ProxyFetcher::Providers::Proxypedia do + before :all do + ProxyFetcher.config.provider = :proxypedia + end + + it_behaves_like "a manager" +end