Merge pull request #36 from nbulaj/add-providers

Add new providers

CHANGELOG.md

@@ -4,7 +4,8 @@ Reverse Chronological Order:
 
 ## `master`
 
-* Add your description here
+* Add MTPro provider
+* Add Proxypedia provider
 
 ## `0.13.0` (2020-03-09)
 

Gemfile

@@ -6,7 +6,7 @@ gemspec
 
 gem "nokogiri", "~> 1.8"
 gem "oga", "~> 3.2"
-gem "rubocop", "~> 0.74"
+gem "rubocop", "~> 0.80"
 
 group :test do
  gem "coveralls", require: false

lib/proxy_fetcher.rb

@@ -33,10 +33,12 @@ module ProxyFetcher
  # ProxyFetcher providers namespace
  module Providers
  require File.dirname(__FILE__) + "/proxy_fetcher/providers/base"
+ require File.dirname(__FILE__) + "/proxy_fetcher/providers/proxypedia"
  require File.dirname(__FILE__) + "/proxy_fetcher/providers/free_proxy_list"
  require File.dirname(__FILE__) + "/proxy_fetcher/providers/free_proxy_list_ssl"
  require File.dirname(__FILE__) + "/proxy_fetcher/providers/gather_proxy"
  require File.dirname(__FILE__) + "/proxy_fetcher/providers/http_tunnel"
+ require File.dirname(__FILE__) + "/proxy_fetcher/providers/mtpro"
  require File.dirname(__FILE__) + "/proxy_fetcher/providers/proxy_list"
  require File.dirname(__FILE__) + "/proxy_fetcher/providers/xroxy"
  end

lib/proxy_fetcher/providers/free_proxy_list.rb

@@ -45,27 +45,6 @@ def to_proxy(html_node)
  #
  def parse_type(html_node)
  https = html_node.content_at("td[6]")
- # frozen_string_literal: true
- # FreeProxyList provider class.
- # Provider URL to fetch proxy list
- # [NOTE] Doesn't support filtering
- # Converts HTML node (entry of N tags) to <code>ProxyFetcher::Proxy</code>
- # object.
- #
- # @param html_node [Object]
- # HTML node from the <code>ProxyFetcher::Document</code> DOM model.
- #
- # @return [ProxyFetcher::Proxy]
- # Proxy object
- #
- # Parses HTML node to extract proxy type.
- #
- # @param html_node [Object]
- # HTML node from the <code>ProxyFetcher::Document</code> DOM model.
- #
- # @return [String]
- # Proxy type
- #
  https&.casecmp("yes")&.zero? ? ProxyFetcher::Proxy::HTTPS : ProxyFetcher::Proxy::HTTP
  end
  end

lib/proxy_fetcher/providers/mtpro.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require "json"
+
+module ProxyFetcher
+ module Providers
+ # GatherProxy provider class.
+ class MTPro < Base
+ # Provider URL to fetch proxy list
+ def provider_url
+ "https://mtpro.xyz/api/?type=socks"
+ end
+
+ def load_proxy_list(filters = {})
+ html = load_html(provider_url, filters)
+ JSON.parse(html)
+ rescue JSON::ParserError
+ []
+ end
+
+ # Converts HTML node (entry of N tags) to <code>ProxyFetcher::Proxy</code>
+ # object.
+ #
+ # @param node [Object]
+ # HTML node from the <code>ProxyFetcher::Document</code> DOM model.
+ #
+ # @return [ProxyFetcher::Proxy]
+ # Proxy object
+ #
+ def to_proxy(node)
+ ProxyFetcher::Proxy.new.tap do |proxy|
+ proxy.addr = node["ip"]
+ proxy.port = Integer(node["port"])
+ proxy.country = node["country"]
+ proxy.anonymity = "Unknown"
+ proxy.type = ProxyFetcher::Proxy::SOCKS5
+ end
+ end
+ end
+
+ ProxyFetcher::Configuration.register_provider(:mtpro, MTPro)
+ end
+end

lib/proxy_fetcher/providers/proxypedia.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module ProxyFetcher
+ module Providers
+ # FreeProxyList provider class.
+ class Proxypedia < Base
+ # Provider URL to fetch proxy list
+ def provider_url
+ "https://proxypedia.org"
+ end
+
+ # [NOTE] Doesn't support filtering
+ def xpath
+ "//main/ul/li[position()>1]"
+ end
+
+ # Converts HTML node (entry of N tags) to <code>ProxyFetcher::Proxy</code>
+ # object.]
+ #
+ # @param html_node [Object]
+ # HTML node from the <code>ProxyFetcher::Document</code> DOM model.
+ #
+ # @return [ProxyFetcher::Proxy]
+ # Proxy object
+ #
+ def to_proxy(html_node)
+ addr, port = html_node.content_at("a").to_s.split(":")
+
+ ProxyFetcher::Proxy.new.tap do |proxy|
+ proxy.addr = addr
+ proxy.port = Integer(port)
+ proxy.country = parse_country(html_node)
+ proxy.anonymity = "Unknown"
+ proxy.type = ProxyFetcher::Proxy::HTTP
+ end
+ end
+
+ private
+
+ def parse_country(html_node)
+ text = html_node.content.to_s
+ text[/\((.+?)\)/, 1] || "Unknown"
+ end
+ end
+
+ ProxyFetcher::Configuration.register_provider(:proxypedia, Proxypedia)
+ end
+end

spec/proxy_fetcher/providers/mtpro_spec.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require "spec_helper"
+
+describe ProxyFetcher::Providers::MTPro do
+ before :all do
+ ProxyFetcher.config.provider = :mtpro
+ end
+
+ it_behaves_like "a manager"
+end

spec/proxy_fetcher/providers/proxypedia_spec.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require "spec_helper"
+
+describe ProxyFetcher::Providers::Proxypedia do
+ before :all do
+ ProxyFetcher.config.provider = :proxypedia
+ end
+
+ it_behaves_like "a manager"
+end