-
Notifications
You must be signed in to change notification settings - Fork 4
/
web.go
113 lines (99 loc) · 2.14 KB
/
web.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
package gopwn
import (
"crypto/tls"
"io"
"io/ioutil"
"net/http"
"net/url"
"os"
"time"
)
type HTTPClientOptions struct {
Timeout time.Duration
ProxyURL string
TLSClientConfig *tls.Config
Cookie *http.Cookie
Headers map[string]string
UserAgent string
}
func HTTPGet(url string, optFns ...func(o *HTTPClientOptions)) ([]byte, error) {
options := HTTPClientOptions{
Timeout: 5 * time.Second,
}
for _, fn := range optFns {
fn(&options)
}
client := newHTTPClient(options)
req, err := newHTTPRequest("GET", url, options)
if err != nil {
return nil, err
}
res, err := client.Do(req)
if err != nil {
return nil, err
}
defer res.Body.Close()
content, err := ioutil.ReadAll(res.Body)
if err != nil {
return nil, err
}
return content, nil
}
func Download(url, filename string, optFns ...func(o *HTTPClientOptions)) error {
options := HTTPClientOptions{
Timeout: 10 * time.Second,
}
for _, fn := range optFns {
fn(&options)
}
client := newHTTPClient(options)
req, err := newHTTPRequest("GET", url, options)
if err != nil {
return err
}
res, err := client.Do(req)
if err != nil {
return err
}
defer res.Body.Close()
out, err := os.Create(filename)
if err != nil {
return err
}
defer out.Close()
_, err = io.Copy(out, res.Body)
if err != nil {
return err
}
return nil
}
func newHTTPClient(options HTTPClientOptions) *http.Client {
transport := http.DefaultTransport.(*http.Transport).Clone()
transport.TLSClientConfig = options.TLSClientConfig
if options.ProxyURL != "" {
proxyURL, _ := url.Parse(options.ProxyURL)
transport.Proxy = http.ProxyURL(proxyURL)
}
return &http.Client{
Timeout: options.Timeout,
Transport: transport,
}
}
func newHTTPRequest(method, url string, options HTTPClientOptions) (*http.Request, error) {
r, err := http.NewRequest(method, url, nil)
if err != nil {
return nil, err
}
if options.Cookie != nil {
r.AddCookie(options.Cookie)
}
if options.UserAgent != "" {
r.Header.Set("User-Agent", options.UserAgent)
}
if len(options.Headers) > 0 {
for k, v := range options.Headers {
r.Header.Set(k, v)
}
}
return r, nil
}