Merge pull request #26 from nbulaj/fix-proxy-docker

Fix ProxyDocker provider

CHANGELOG.md

@@ -6,6 +6,11 @@ Reverse Chronological Order:
 
 * Add your description here
 
+## `0.10.1` (2019-03-07)
+
+* Fix broken ProxyDocker provider.
+* Refactor gem internals.
+
 ## `0.9.0` (2019-01-22)
 
 * Fix a problem with stuck of proxies list loading.

bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+require 'proxy_fetcher'
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require 'irb'
+IRB.start(__FILE__)

lib/proxy_fetcher/configuration.rb

@@ -35,7 +35,11 @@ class Configuration
 
  # @!attribute [r] adapter
  # @return [Object] HTML parser adapter
- attr_reader :adapter
+ attr_accessor :adapter
+
+ # @!attribute [r] adapter_class
+ # @return [Object] HTML adapter class
+ attr_reader :adapter_class
 
  # @!attribute [r] http_client
  # @return [Object] HTTP client class
@@ -120,14 +124,11 @@ def reset!
  self.providers = self.class.registered_providers
  end
 
- # Setups HTML parser adapter for all the proxy providers.
- #
- # @param name_or_class [String, Symbol, Class]
- # name of the adapter or it's class
- #
- def adapter=(name_or_class)
- @adapter = ProxyFetcher::Document::Adapters.lookup(name_or_class)
- @adapter.setup!
+ def adapter_class
+ return @adapter_class if defined?(@adapter_class)
+
+ @adapter_class = ProxyFetcher::Document::Adapters.lookup(adapter)
+ @adapter_class.setup!
  end
 
  # Setups collection of providers that will be used to fetch proxies.

lib/proxy_fetcher/document.rb

@@ -17,7 +17,7 @@ class Document
  # ProxyFetcher document model
  #
  def self.parse(data)
- new(ProxyFetcher.config.adapter.parse(data))
+ new(ProxyFetcher.config.adapter_class.parse(data))
  end
 
  # Initialize abstract ProxyFetcher HTML Document

lib/proxy_fetcher/document/adapters.rb

@@ -30,7 +30,7 @@ def lookup(name_or_class)
 
  case name_or_class
  when Symbol, String
- adapter_name = name_or_class.to_s.capitalize << ADAPTER
+ adapter_name = "#{name_or_class.to_s.capitalize}#{ADAPTER}"
  ProxyFetcher::Document.const_get(adapter_name)
  else
  name_or_class

lib/proxy_fetcher/document/adapters/abstract_adapter.rb

@@ -52,6 +52,7 @@ def proxy_node
  #
  def self.setup!(*args)
  install_requirements!(*args)
+ self
  rescue LoadError => error
  raise Exceptions::AdapterSetupError.new(name, error.message)
  end

lib/proxy_fetcher/providers/base.rb

@@ -74,7 +74,10 @@ def load_document(url, filters = {})
  def build_proxy(*args)
  to_proxy(*args)
  rescue StandardError => error
- ProxyFetcher.logger.warn("Failed to build Proxy object due to error: #{error.message}")
+ ProxyFetcher.logger.warn(
+ "Failed to build Proxy object for #{self.class.name} due to error: #{error.message}"
+ )
+
  nil
  end
 

lib/proxy_fetcher/providers/proxy_docker.rb

@@ -6,6 +6,8 @@ module ProxyFetcher
  module Providers
  # ProxyDocker provider class.
  class ProxyDocker < Base
+ attr_reader :cookie, :token
+
  # Provider URL to fetch proxy list
  def provider_url
  'https://www.proxydocker.com/en/api/proxylist/'
@@ -17,7 +19,7 @@ def provider_method
 
  def provider_params
  {
- token: 'GmZyl0OJmmgrWakdzO7AFf6AWfkdledR6xmKvGmwmJg',
+ token: @token,
  country: 'all',
  city: 'all',
  state: 'all',
@@ -31,7 +33,7 @@ def provider_params
 
  def provider_headers
  {
- cookie: 'PHPSESSID=7f59558ee58b1e4352c4ab4c2f1a3c11'
+ cookie: @cookie
  }
  end
 
@@ -44,6 +46,8 @@ def provider_headers
  #
  # [NOTE] Doesn't support direct filters
  def load_proxy_list(*)
+ load_dependencies
+
  json = JSON.parse(load_html(provider_url, {}))
  json.fetch('proxies', [])
  rescue JSON::ParserError
@@ -70,6 +74,8 @@ def to_proxy(node)
  end
  end
 
+ private
+
  def types_mapping
  {
  '16' => ProxyFetcher::Proxy::HTTP,
@@ -80,6 +86,24 @@ def types_mapping
  '6' => ProxyFetcher::Proxy::HTTP # CON80
  }
  end
+
+ def load_dependencies
+ client = ProxyFetcher.config.http_client.new('https://www.proxydocker.com')
+ response = client.fetch_with_headers
+
+ @cookie = load_cookie_from(response)
+ @token = load_token_from(response)
+ end
+
+ def load_cookie_from(response)
+ cookie_headers = (response.headers['Set-Cookie'] || [])
+ cookie_headers.join('; ')
+ end
+
+ def load_token_from(response)
+ html = response.body.to_s
+ html[/meta\s+name\s*=["']_token["']\s+content.+["'](.+?)["']\s*>/i, 1]
+ end
  end
 
  ProxyFetcher::Configuration.register_provider(:proxy_docker, ProxyDocker)

lib/proxy_fetcher/utils/http_client.rb

@@ -68,21 +68,32 @@ def initialize(url, method: :get, params: {}, headers: {})
  # response body
  #
  def fetch
- # TODO: must be more generic
- response = if method == :post
- http.post(url, form: params, ssl_context: ssl_ctx)
- else
- http.get(url, ssl_context: ssl_ctx)
- end
-
+ response = process_http_request
  response.body.to_s
  rescue StandardError => error
- ProxyFetcher.logger.warn("Failed to load proxy list for #{url} (#{error.message})")
+ ProxyFetcher.logger.warn("Failed to process request to #{url} (#{error.message})")
  ''
  end
 
+ def fetch_with_headers
+ process_http_request
+ rescue StandardError => error
+ ProxyFetcher.logger.warn("Failed to process request to #{url} (#{error.message})")
+ HTTP::Response.new(version: '1.1', status: 500, body: '')
+ end
+
  protected
 
+ def process_http_request(http_method: method, http_params: params)
+ raise ArgumentError, 'wrong http method name!' unless HTTP::Request::METHODS.include?(http_method)
+
+ http.public_send(
+ http_method.to_sym, url,
+ form: http_params,
+ ssl_context: ssl_ctx
+ )
+ end
+
  # Default HTTP client headers
  #
  # @return [Hash]

lib/proxy_fetcher/version.rb

@@ -15,7 +15,7 @@ module VERSION
  # Minor version number
  MINOR = 10
  # Smallest version number
- TINY = 0
+ TINY = 1
 
  # Full version number
  STRING = [MAJOR, MINOR, TINY].compact.join('.')

spec/proxy_fetcher/configuration_spec.rb

@@ -68,7 +68,9 @@ def self.install_requirements!
  end
  end
 
- expect { ProxyFetcher.config.adapter = CustomAdapter }
+ ProxyFetcher.config.adapter = CustomAdapter
+
+ expect { ProxyFetcher::Manager.new }
  .to raise_error(ProxyFetcher::Exceptions::AdapterSetupError)
 
  ProxyFetcher.instance_variable_set('@config', old_config)

spec/proxy_fetcher/providers/base_spec.rb

@@ -44,7 +44,7 @@ def load_proxy_list(*)
 
  allow_any_instance_of(HTTP::Client).to receive(:get).and_raise(StandardError)
 
- expect(logger).to receive(:warn).with(/Failed to load proxy list for http[s:\/]/)
+ expect(logger).to receive(:warn).with(/Failed to process request to http[s:\/]/)
 
  ProxyFetcher::Manager.new
  end