From fb509856cd46f047cb36a2e5e73eb62a808b78c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Trojnara?= Date: Sun, 10 Oct 2021 15:27:45 +0200 Subject: [PATCH] relese 1.0 --- .gitignore | 2 + README.md | 51 ++++++++- dist.sh | 9 ++ torget.go | 297 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 358 insertions(+), 1 deletion(-) create mode 100755 dist.sh create mode 100644 torget.go diff --git a/.gitignore b/.gitignore index 66fd13c..6b18d81 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,8 @@ *.dll *.so *.dylib +/dist/ +/torget # Test binary, built with `go test -c` *.test diff --git a/README.md b/README.md index 992e353..019a0e8 100644 --- a/README.md +++ b/README.md @@ -1 +1,50 @@ -# torget \ No newline at end of file +# torget + + $ go build + $ ./torget -h + torget 1.0, a fast large file downloader over locally installed Tor + Copyright © 2021 Michał Trojnara + Licensed under GNU/GPL version 3 + + Usage: torget [-c circuits] url + -c int + concurrent circuits (default 20) + $ ./torget https://ftp.nluug.nl/os/Linux/distr/tails/tails/stable/tails-amd64-4.23/tails-amd64-4.23.img + Output file: tails-amd64-4.23.img + Download length: 1185939456 bytes + 0.00% done, stalled + 0.05% done, 600.00 KB/s, ETA 0:32:55 + 0.11% done, 752.00 KB/s, ETA 0:26:15 + 0.23% done, 1.38 MB/s, ETA 0:14:14 + 0.59% done, 4.23 MB/s, ETA 0:04:38 + 0.80% done, 2.50 MB/s, ETA 0:07:49 + 1.05% done, 3.02 MB/s, ETA 0:06:29 + 1.28% done, 2.69 MB/s, ETA 0:07:15 + 1.76% done, 5.64 MB/s, ETA 0:03:26 + 2.11% done, 4.22 MB/s, ETA 0:04:35 + 2.42% done, 3.70 MB/s, ETA 0:05:12 + 2.82% done, 4.75 MB/s, ETA 0:04:02 + 3.36% done, 6.36 MB/s, ETA 0:03:00 + Get https://ftp.nluug.nl/os/Linux/distr/tails/tails/stable/tails-amd64-4.23/tails-amd64-4.23.img: context canceled + 3.93% done, 6.78 MB/s, ETA 0:02:48 + 4.64% done, 8.46 MB/s, ETA 0:02:13 + 5.14% done, 5.88 MB/s, ETA 0:03:11 + ... + 98.28% done, 3.29 MB/s, ETA 0:00:06 + 98.52% done, 2.84 MB/s, ETA 0:00:06 + 98.68% done, 1.88 MB/s, ETA 0:00:08 + 98.88% done, 2.41 MB/s, ETA 0:00:05 + 99.11% done, 2.73 MB/s, ETA 0:00:03 + 99.28% done, 2.01 MB/s, ETA 0:00:04 + 99.44% done, 1.88 MB/s, ETA 0:00:03 + 99.64% done, 2.40 MB/s, ETA 0:00:01 + 99.71% done, 824.14 KB/s, ETA 0:00:04 + 99.76% done, 621.65 KB/s, ETA 0:00:04 + 99.84% done, 906.99 KB/s, ETA 0:00:02 + 99.88% done, 499.65 KB/s, ETA 0:00:02 + 99.92% done, 475.74 KB/s, ETA 0:00:01 + 99.95% done, 313.94 KB/s, ETA 0:00:01 + 99.96% done, 175.01 KB/s, ETA 0:00:02 + 99.98% done, 206.95 KB/s, ETA 0:00:01 + 99.99% done, 176.03 KB/s, ETA 0:00:00 + 100.00% done, 17.71 KB/s, ETA 0:00:02 diff --git a/dist.sh b/dist.sh new file mode 100755 index 0000000..ccfdca5 --- /dev/null +++ b/dist.sh @@ -0,0 +1,9 @@ +#!/bin/sh +set -e + +mkdir -p dist +for OS in windows linux freebsd openbsd netbsd; do + echo "Building for $OS" + GOOS=$OS GOARCH=amd64 go build -o "dist/torget-$OS" -ldflags '-w -s' +done +mv dist/torget-windows dist/torget.exe diff --git a/torget.go b/torget.go new file mode 100644 index 0000000..49fd2a4 --- /dev/null +++ b/torget.go @@ -0,0 +1,297 @@ +package main + +/* + torget 1.0, a fast large file downloader over locally installed Tor + Copyright © 2021 Michał Trojnara + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +*/ + +import ( + "context" + "flag" + "fmt" + "io" + "net/http" + "net/url" + "os" + "strings" + "sync" + "time" +) + +type chunk struct { + start int64 + length int64 + circuit int +} + +type State struct { + src string + dst string + total int64 + circuits int + chunks []chunk + done chan int + mutex sync.Mutex +} + +const torBlock = 8000 // the longest plain text block in Tor + +func NewState(circuits int) *State { + var s State + s.circuits = circuits + s.chunks = make([]chunk, s.circuits) + s.done = make(chan int) + return &s +} + +func (s *State) fetchChunk(id int) { + defer func() { + s.done <- id + }() + s.mutex.Lock() + start := s.chunks[id].start + length := s.chunks[id].length + s.mutex.Unlock() + if length == 0 { + return + } + + // make an HTTP request in a new circuit + ctx, cancel := context.WithCancel(context.TODO()) + timer := time.AfterFunc(10*time.Second, func() { + cancel() + }) + defer func() { + if timer.Stop() { + cancel() // make sure cancel() is executed exactly once + } + }() + user := fmt.Sprintf("tg%d", s.chunks[id].circuit) + proxyUrl, _ := url.Parse("socks5://" + user + ":" + user + "@127.0.0.1:9050/") + client := &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(proxyUrl)}} + req, _ := http.NewRequestWithContext(ctx, "GET", s.src, nil) + header := fmt.Sprintf("bytes=%d-%d", start, start+length-1) + req.Header.Add("Range", header) + resp, err := client.Do(req) + if err != nil { + fmt.Println(err.Error()) + return + } + if resp.Body == nil { + fmt.Println("No response body") + return + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusPartialContent { + fmt.Println("No response body") + return + } + + // open the output file + file, err := os.OpenFile(s.dst, os.O_WRONLY, 0) + defer file.Close() + if err != nil { + fmt.Println(err.Error()) + return + } + _, err = file.Seek(start, io.SeekStart) + if err != nil { + fmt.Println(err.Error()) + return + } + + // copy network data to the output file + buffer := make([]byte, torBlock) + for { + if !timer.Stop() { // cancel() already started + return + } + timer.Reset(3 * time.Second) + n, err := resp.Body.Read(buffer) + // fmt.Println("writing", id, n) + if n > 0 { + file.Write(buffer[:n]) + s.mutex.Lock() + if int64(n) < s.chunks[id].length { + s.chunks[id].start += int64(n) + s.chunks[id].length -= int64(n) + } else { + s.chunks[id].length = 0 + } + s.mutex.Unlock() + if s.chunks[id].length == 0 { + break + } + } + if err != nil { + fmt.Println("Downloading:", err.Error()) + break + } + } +} + +func (s *State) progress() { + var prev int64 + for { + time.Sleep(time.Second) + curr := s.total + s.mutex.Lock() + for id := 0; id < s.circuits; id++ { + curr -= s.chunks[id].length + } + s.mutex.Unlock() + if curr == prev { + fmt.Printf("%6.2f%% done, stalled\n", + 100*float32(curr)/float32(s.total)) + } else { + speed := float32(curr-prev) / 1000 + prefix := "K" + if speed >= 1000 { + speed /= 1000 + prefix = "M" + } + if speed >= 1000 { + speed /= 1000 + prefix = "G" + } + seconds := (s.total - curr) / (curr - prev) + fmt.Printf("%6.2f%% done, %6.2f %sB/s, ETA %d:%02d:%02d\n", + 100*float32(curr)/float32(s.total), + speed, prefix, + seconds/3600, seconds/60%60, seconds%60) + } + prev = curr + } +} + +func (s *State) Fetch(src string) int { + // setup file name + s.src = src + srcUrl, err := url.Parse(src) + if err != nil { + fmt.Println(err.Error()) + return 1 + } + path := srcUrl.EscapedPath() + slash := strings.LastIndex(path, "/") + if slash >= 0 { + s.dst = path[slash+1:] + } else { + s.dst = path + } + if s.dst == "" { + s.dst = "index" + } + fmt.Println("Output file:", s.dst) + + // get the target length + proxyUrl, _ := url.Parse("socks5://torget:torget@127.0.0.1:9050/") + client := &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(proxyUrl)}} + resp, err := client.Head(s.src) + if err != nil { + fmt.Println(err.Error()) + return 1 + } + s.total = resp.ContentLength + if s.total <= 0 { + fmt.Println("Failed to retrieve download length") + return 1 + } + fmt.Println("Download length:", s.total, "bytes") + + // create the output file + file, err := os.Create(s.dst) + if file != nil { + file.Close() + } + if err != nil { + fmt.Println(err.Error()) + return 1 + } + + // initialize chunks + chunkLen := s.total / int64(s.circuits) + seq := 0 + for id := 0; id < s.circuits; id++ { + s.chunks[id].start = int64(id) * chunkLen + s.chunks[id].length = chunkLen + s.chunks[id].circuit = seq + seq++ + } + s.chunks[s.circuits-1].length += s.total % int64(s.circuits) + + // spawn initial fetchers + go s.progress() + for id := 0; id < s.circuits; id++ { + go s.fetchChunk(id) + time.Sleep(499 * time.Millisecond) // be gentle to the local tor daemon + } + + // spawn additional fetchers as needed + for { + id := <-s.done + if s.chunks[id].length > 0 { // error + // resume in a new and hopefully faster circuit + s.chunks[id].circuit = seq + seq++ + // fmt.Println("resume", s.chunks[id].length) + } else { // completed + longest := 0 + s.mutex.Lock() + for i := 1; i < s.circuits; i++ { + if s.chunks[i].length > s.chunks[longest].length { + longest = i + } + } + s.mutex.Unlock() + // fmt.Println("completed", s.chunks[longest].length) + if s.chunks[longest].length == 0 { // all done + break + } + if s.chunks[longest].length <= 5*torBlock { // too short to split + continue + } + // this circuit is faster, so we split 80%/20% + s.mutex.Lock() + s.chunks[id].length = s.chunks[longest].length * 4 / 5 + s.chunks[longest].length -= s.chunks[id].length + s.chunks[id].start = s.chunks[longest].start + s.chunks[longest].length + s.mutex.Unlock() + } + go s.fetchChunk(id) + } + return 0 +} + +func main() { + circuits := flag.Int("c", 20, "concurrent circuits") + flag.Usage = func() { + fmt.Fprintln(os.Stderr, "torget 1.0, a fast large file downloader over locally installed Tor") + fmt.Fprintln(os.Stderr, "Copyright © 2021 Michał Trojnara ") + fmt.Fprintln(os.Stderr, "Licensed under GNU/GPL version 3") + fmt.Fprintln(os.Stderr) + fmt.Fprintln(os.Stderr, "Usage: torget [-c circuits] url") + flag.PrintDefaults() + } + flag.Parse() + if flag.NArg() != 1 { + flag.Usage() + os.Exit(1) + } + state := NewState(*circuits) + os.Exit(state.Fetch(flag.Arg(0))) +} + +// vim: noet:ts=4:sw=4:sts=4:spell