From 903231e6123ce340548b40924792aa9e3a4c6702 Mon Sep 17 00:00:00 2001 From: Nikita Bulai Date: Fri, 15 Mar 2019 10:48:50 +0300 Subject: [PATCH] Remove ProxyDocker provider, bump version --- CHANGELOG.md | 4 + README.md | 4 +- lib/proxy_fetcher.rb | 1 - lib/proxy_fetcher/configuration.rb | 6 + .../configuration/providers_registry.rb | 2 +- .../document/adapters/abstract_adapter.rb | 2 +- lib/proxy_fetcher/providers/proxy_docker.rb | 111 ------------------ lib/proxy_fetcher/version.rb | 2 +- spec/proxy_fetcher/configuration_spec.rb | 6 +- .../providers/proxy_docker_spec.rb | 11 -- 10 files changed, 18 insertions(+), 131 deletions(-) delete mode 100644 lib/proxy_fetcher/providers/proxy_docker.rb delete mode 100644 spec/proxy_fetcher/providers/proxy_docker_spec.rb diff --git a/CHANGELOG.md b/CHANGELOG.md index c4e00f3..bb40737 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ Reverse Chronological Order: * Add your description here +## `0.10.2` (2019-03-15) + +* Remove ProxyDocker provider (no longer workable) + ## `0.10.1` (2019-03-07) * Fix broken ProxyDocker provider. diff --git a/README.md b/README.md index 64da231..8924ba5 100644 --- a/README.md +++ b/README.md @@ -145,7 +145,7 @@ If you need to filter proxy list, for example, by country or response time and * with GET params, then you can just pass your filters like a simple Ruby hash to the Manager instance: ```ruby -ProxyFetcher.config.providers = :proxy_docker +ProxyFetcher.config.providers = :xroxy manager = ProxyFetcher::Manager.new(filters: { country: 'PL', maxtime: '500' }) manager.proxies @@ -199,7 +199,7 @@ proxy_fetcher >> proxies.txt # Will download proxies from the default provider, If you need a list of proxies from some specific provider, then you need to pass it's name with `-p` option: ```bash -proxy_fetcher -p proxy_docker >> proxies.txt # Will download proxies from the default provider, validate them and write to file +proxy_fetcher -p xroxy >> proxies.txt # Will download proxies from the default provider, validate them and write to file ``` If you need a list of proxies in JSON format just pass a `--json` option to the command: diff --git a/lib/proxy_fetcher.rb b/lib/proxy_fetcher.rb index ca1be13..f353e42 100644 --- a/lib/proxy_fetcher.rb +++ b/lib/proxy_fetcher.rb @@ -37,7 +37,6 @@ module Providers require File.dirname(__FILE__) + '/proxy_fetcher/providers/free_proxy_list_ssl' require File.dirname(__FILE__) + '/proxy_fetcher/providers/gather_proxy' require File.dirname(__FILE__) + '/proxy_fetcher/providers/http_tunnel' - require File.dirname(__FILE__) + '/proxy_fetcher/providers/proxy_docker' require File.dirname(__FILE__) + '/proxy_fetcher/providers/proxy_list' require File.dirname(__FILE__) + '/proxy_fetcher/providers/xroxy' end diff --git a/lib/proxy_fetcher/configuration.rb b/lib/proxy_fetcher/configuration.rb index 1d90dc5..6c1283e 100644 --- a/lib/proxy_fetcher/configuration.rb +++ b/lib/proxy_fetcher/configuration.rb @@ -124,11 +124,17 @@ def reset! self.providers = self.class.registered_providers end + def adapter=(value) + remove_instance_variable(:@adapter_class) if defined?(@adapter_class) + @adapter = value + end + def adapter_class return @adapter_class if defined?(@adapter_class) @adapter_class = ProxyFetcher::Document::Adapters.lookup(adapter) @adapter_class.setup! + @adapter_class end # Setups collection of providers that will be used to fetch proxies. diff --git a/lib/proxy_fetcher/configuration/providers_registry.rb b/lib/proxy_fetcher/configuration/providers_registry.rb index b5a53e6..55f9fc9 100644 --- a/lib/proxy_fetcher/configuration/providers_registry.rb +++ b/lib/proxy_fetcher/configuration/providers_registry.rb @@ -14,7 +14,7 @@ def providers end # Add custom provider to common registry. - # Requires proxy provider name ('proxy_docker' for example) and a class + # Requires proxy provider name ('xroxy' for example) and a class # that implements the parsing logic. # # @param name [String, Symbol] diff --git a/lib/proxy_fetcher/document/adapters/abstract_adapter.rb b/lib/proxy_fetcher/document/adapters/abstract_adapter.rb index 8684709..7af1a74 100644 --- a/lib/proxy_fetcher/document/adapters/abstract_adapter.rb +++ b/lib/proxy_fetcher/document/adapters/abstract_adapter.rb @@ -53,7 +53,7 @@ def proxy_node def self.setup!(*args) install_requirements!(*args) self - rescue LoadError => error + rescue LoadError, StandardError => error raise Exceptions::AdapterSetupError.new(name, error.message) end end diff --git a/lib/proxy_fetcher/providers/proxy_docker.rb b/lib/proxy_fetcher/providers/proxy_docker.rb deleted file mode 100644 index 1f494b1..0000000 --- a/lib/proxy_fetcher/providers/proxy_docker.rb +++ /dev/null @@ -1,111 +0,0 @@ -# frozen_string_literal: true - -require 'json' - -module ProxyFetcher - module Providers - # ProxyDocker provider class. - class ProxyDocker < Base - attr_reader :cookie, :token - - # Provider URL to fetch proxy list - def provider_url - 'https://www.proxydocker.com/en/api/proxylist/' - end - - def provider_method - :post - end - - def provider_params - { - token: @token, - country: 'all', - city: 'all', - state: 'all', - port: 'all', - type: 'all', - anonymity: 'all', - need: 'all', - page: '1' - } - end - - def provider_headers - { - cookie: @cookie - } - end - - # Fetches HTML content by sending HTTP request to the provider URL and - # parses the document (built as abstract ProxyFetcher::Document) - # to return all the proxy entries (HTML nodes). - # - # @return [Array] - # Collection of extracted HTML nodes with full proxy info - # - # [NOTE] Doesn't support direct filters - def load_proxy_list(*) - load_dependencies - - json = JSON.parse(load_html(provider_url, {})) - json.fetch('proxies', []) - rescue JSON::ParserError - [] - end - - # Converts JSON node to ProxyFetcher::Proxy - # object. - # - # @param node [Hash] - # JSON entry from the API response - # - # @return [ProxyFetcher::Proxy] - # Proxy object - # - def to_proxy(node) - ProxyFetcher::Proxy.new.tap do |proxy| - proxy.addr = node['ip'] - proxy.port = node['port'] - - proxy.type = types_mapping.fetch(node['type'], ProxyFetcher::Proxy::HTTP) - proxy.anonymity = "Lvl#{node['anonymity']}" - proxy.country = node['country'] - end - end - - private - - def types_mapping - { - '16' => ProxyFetcher::Proxy::HTTP, - '26' => ProxyFetcher::Proxy::HTTPS, - '3' => ProxyFetcher::Proxy::SOCKS4, - '4' => ProxyFetcher::Proxy::SOCKS5, - '56' => ProxyFetcher::Proxy::HTTP, # CON25 - '6' => ProxyFetcher::Proxy::HTTP # CON80 - } - end - - def load_dependencies - client = ProxyFetcher.config.http_client.new('https://www.proxydocker.com') - response = client.fetch_with_headers - - @cookie = load_cookie_from(response) - @token = load_token_from(response) - end - - def load_cookie_from(response) - cookie_headers = (response.headers['Set-Cookie'] || []) - cookie_headers.join('; ') - end - - def load_token_from(response) - html = response.body.to_s - html[/meta\s+name\s*=["']_token["']\s+content.+["'](.+?)["']\s*>/i, 1] - end - end - - ProxyFetcher::Configuration.register_provider(:proxy_docker, ProxyDocker) - end -end diff --git a/lib/proxy_fetcher/version.rb b/lib/proxy_fetcher/version.rb index 3e330dd..7d91195 100644 --- a/lib/proxy_fetcher/version.rb +++ b/lib/proxy_fetcher/version.rb @@ -15,7 +15,7 @@ module VERSION # Minor version number MINOR = 10 # Smallest version number - TINY = 1 + TINY = 2 # Full version number STRING = [MAJOR, MINOR, TINY].compact.join('.') diff --git a/spec/proxy_fetcher/configuration_spec.rb b/spec/proxy_fetcher/configuration_spec.rb index f775293..a6ccbcf 100644 --- a/spec/proxy_fetcher/configuration_spec.rb +++ b/spec/proxy_fetcher/configuration_spec.rb @@ -60,7 +60,7 @@ def self.connectable?(*) context 'custom HTML parsing adapter' do it "fails if adapter can't be installed" do - old_config = ProxyFetcher.config.dup + old_adapter = ProxyFetcher.config.adapter.dup class CustomAdapter < ProxyFetcher::Document::AbstractAdapter def self.install_requirements! @@ -70,10 +70,10 @@ def self.install_requirements! ProxyFetcher.config.adapter = CustomAdapter - expect { ProxyFetcher::Manager.new } + expect { ProxyFetcher.config.adapter_class } .to raise_error(ProxyFetcher::Exceptions::AdapterSetupError) - ProxyFetcher.instance_variable_set('@config', old_config) + ProxyFetcher.config.adapter = old_adapter end end end diff --git a/spec/proxy_fetcher/providers/proxy_docker_spec.rb b/spec/proxy_fetcher/providers/proxy_docker_spec.rb deleted file mode 100644 index 6ecfe74..0000000 --- a/spec/proxy_fetcher/providers/proxy_docker_spec.rb +++ /dev/null @@ -1,11 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -describe ProxyFetcher::Providers::ProxyDocker do - before :all do - ProxyFetcher.config.provider = :proxy_docker - end - - it_behaves_like 'a manager' -end