relese 1.0

.gitignore

@@ -4,6 +4,8 @@
 *.dll
 *.so
 *.dylib
+/dist/
+/torget
 
 # Test binary, built with `go test -c`
 *.test

README.md

@@ -1 +1,50 @@
-# torget
+# torget
+
+    $ go build
+    $ ./torget -h
+    torget 1.0, a fast large file downloader over locally installed Tor
+    Copyright © 2021 Michał Trojnara <[email protected]>
+    Licensed under GNU/GPL version 3
+
+    Usage: torget [-c circuits] url
+      -c int
+            concurrent circuits (default 20)
+    $ ./torget https://ftp.nluug.nl/os/Linux/distr/tails/tails/stable/tails-amd64-4.23/tails-amd64-4.23.img
+    Output file: tails-amd64-4.23.img
+    Download length: 1185939456 bytes
+      0.00% done, stalled
+      0.05% done, 600.00 KB/s, ETA 0:32:55
+      0.11% done, 752.00 KB/s, ETA 0:26:15
+      0.23% done,   1.38 MB/s, ETA 0:14:14
+      0.59% done,   4.23 MB/s, ETA 0:04:38
+      0.80% done,   2.50 MB/s, ETA 0:07:49
+      1.05% done,   3.02 MB/s, ETA 0:06:29
+      1.28% done,   2.69 MB/s, ETA 0:07:15
+      1.76% done,   5.64 MB/s, ETA 0:03:26
+      2.11% done,   4.22 MB/s, ETA 0:04:35
+      2.42% done,   3.70 MB/s, ETA 0:05:12
+      2.82% done,   4.75 MB/s, ETA 0:04:02
+      3.36% done,   6.36 MB/s, ETA 0:03:00
+    Get https://ftp.nluug.nl/os/Linux/distr/tails/tails/stable/tails-amd64-4.23/tails-amd64-4.23.img: context canceled
+      3.93% done,   6.78 MB/s, ETA 0:02:48
+      4.64% done,   8.46 MB/s, ETA 0:02:13
+      5.14% done,   5.88 MB/s, ETA 0:03:11
+    ...
+     98.28% done,   3.29 MB/s, ETA 0:00:06
+     98.52% done,   2.84 MB/s, ETA 0:00:06
+     98.68% done,   1.88 MB/s, ETA 0:00:08
+     98.88% done,   2.41 MB/s, ETA 0:00:05
+     99.11% done,   2.73 MB/s, ETA 0:00:03
+     99.28% done,   2.01 MB/s, ETA 0:00:04
+     99.44% done,   1.88 MB/s, ETA 0:00:03
+     99.64% done,   2.40 MB/s, ETA 0:00:01
+     99.71% done, 824.14 KB/s, ETA 0:00:04
+     99.76% done, 621.65 KB/s, ETA 0:00:04
+     99.84% done, 906.99 KB/s, ETA 0:00:02
+     99.88% done, 499.65 KB/s, ETA 0:00:02
+     99.92% done, 475.74 KB/s, ETA 0:00:01
+     99.95% done, 313.94 KB/s, ETA 0:00:01
+     99.96% done, 175.01 KB/s, ETA 0:00:02
+     99.98% done, 206.95 KB/s, ETA 0:00:01
+     99.99% done, 176.03 KB/s, ETA 0:00:00
+    100.00% done,  17.71 KB/s, ETA 0:00:02

dist.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+set -e
+
+mkdir -p dist
+for OS in windows linux freebsd openbsd netbsd; do
+    echo "Building for $OS"
+    GOOS=$OS GOARCH=amd64 go build -o "dist/torget-$OS" -ldflags '-w -s'
+done
+mv dist/torget-windows dist/torget.exe

torget.go

@@ -0,0 +1,297 @@
+package main
+
+/*
+	torget 1.0, a fast large file downloader over locally installed Tor
+	Copyright © 2021 Michał Trojnara <[email protected]>
+
+	This program is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License.
+
+	This program is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with this program.  If not, see <https://www.gnu.org/licenses/>.
+*/
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
+	"sync"
+	"time"
+)
+
+type chunk struct {
+	start   int64
+	length  int64
+	circuit int
+}
+
+type State struct {
+	src      string
+	dst      string
+	total    int64
+	circuits int
+	chunks   []chunk
+	done     chan int
+	mutex    sync.Mutex
+}
+
+const torBlock = 8000 // the longest plain text block in Tor
+
+func NewState(circuits int) *State {
+	var s State
+	s.circuits = circuits
+	s.chunks = make([]chunk, s.circuits)
+	s.done = make(chan int)
+	return &s
+}
+
+func (s *State) fetchChunk(id int) {
+	defer func() {
+		s.done <- id
+	}()
+	s.mutex.Lock()
+	start := s.chunks[id].start
+	length := s.chunks[id].length
+	s.mutex.Unlock()
+	if length == 0 {
+		return
+	}
+
+	// make an HTTP request in a new circuit
+	ctx, cancel := context.WithCancel(context.TODO())
+	timer := time.AfterFunc(10*time.Second, func() {
+		cancel()
+	})
+	defer func() {
+		if timer.Stop() {
+			cancel() // make sure cancel() is executed exactly once
+		}
+	}()
+	user := fmt.Sprintf("tg%d", s.chunks[id].circuit)
+	proxyUrl, _ := url.Parse("socks5://" + user + ":" + user + "@127.0.0.1:9050/")
+	client := &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(proxyUrl)}}
+	req, _ := http.NewRequestWithContext(ctx, "GET", s.src, nil)
+	header := fmt.Sprintf("bytes=%d-%d", start, start+length-1)
+	req.Header.Add("Range", header)
+	resp, err := client.Do(req)
+	if err != nil {
+		fmt.Println(err.Error())
+		return
+	}
+	if resp.Body == nil {
+		fmt.Println("No response body")
+		return
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusPartialContent {
+		fmt.Println("No response body")
+		return
+	}
+
+	// open the output file
+	file, err := os.OpenFile(s.dst, os.O_WRONLY, 0)
+	defer file.Close()
+	if err != nil {
+		fmt.Println(err.Error())
+		return
+	}
+	_, err = file.Seek(start, io.SeekStart)
+	if err != nil {
+		fmt.Println(err.Error())
+		return
+	}
+
+	// copy network data to the output file
+	buffer := make([]byte, torBlock)
+	for {
+		if !timer.Stop() { // cancel() already started
+			return
+		}
+		timer.Reset(3 * time.Second)
+		n, err := resp.Body.Read(buffer)
+		// fmt.Println("writing", id, n)
+		if n > 0 {
+			file.Write(buffer[:n])
+			s.mutex.Lock()
+			if int64(n) < s.chunks[id].length {
+				s.chunks[id].start += int64(n)
+				s.chunks[id].length -= int64(n)
+			} else {
+				s.chunks[id].length = 0
+			}
+			s.mutex.Unlock()
+			if s.chunks[id].length == 0 {
+				break
+			}
+		}
+		if err != nil {
+			fmt.Println("Downloading:", err.Error())
+			break
+		}
+	}
+}
+
+func (s *State) progress() {
+	var prev int64
+	for {
+		time.Sleep(time.Second)
+		curr := s.total
+		s.mutex.Lock()
+		for id := 0; id < s.circuits; id++ {
+			curr -= s.chunks[id].length
+		}
+		s.mutex.Unlock()
+		if curr == prev {
+			fmt.Printf("%6.2f%% done, stalled\n",
+				100*float32(curr)/float32(s.total))
+		} else {
+			speed := float32(curr-prev) / 1000
+			prefix := "K"
+			if speed >= 1000 {
+				speed /= 1000
+				prefix = "M"
+			}
+			if speed >= 1000 {
+				speed /= 1000
+				prefix = "G"
+			}
+			seconds := (s.total - curr) / (curr - prev)
+			fmt.Printf("%6.2f%% done, %6.2f %sB/s, ETA %d:%02d:%02d\n",
+				100*float32(curr)/float32(s.total),
+				speed, prefix,
+				seconds/3600, seconds/60%60, seconds%60)
+		}
+		prev = curr
+	}
+}
+
+func (s *State) Fetch(src string) int {
+	// setup file name
+	s.src = src
+	srcUrl, err := url.Parse(src)
+	if err != nil {
+		fmt.Println(err.Error())
+		return 1
+	}
+	path := srcUrl.EscapedPath()
+	slash := strings.LastIndex(path, "/")
+	if slash >= 0 {
+		s.dst = path[slash+1:]
+	} else {
+		s.dst = path
+	}
+	if s.dst == "" {
+		s.dst = "index"
+	}
+	fmt.Println("Output file:", s.dst)
+
+	// get the target length
+	proxyUrl, _ := url.Parse("socks5://torget:[email protected]:9050/")
+	client := &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(proxyUrl)}}
+	resp, err := client.Head(s.src)
+	if err != nil {
+		fmt.Println(err.Error())
+		return 1
+	}
+	s.total = resp.ContentLength
+	if s.total <= 0 {
+		fmt.Println("Failed to retrieve download length")
+		return 1
+	}
+	fmt.Println("Download length:", s.total, "bytes")
+
+	// create the output file
+	file, err := os.Create(s.dst)
+	if file != nil {
+		file.Close()
+	}
+	if err != nil {
+		fmt.Println(err.Error())
+		return 1
+	}
+
+	// initialize chunks
+	chunkLen := s.total / int64(s.circuits)
+	seq := 0
+	for id := 0; id < s.circuits; id++ {
+		s.chunks[id].start = int64(id) * chunkLen
+		s.chunks[id].length = chunkLen
+		s.chunks[id].circuit = seq
+		seq++
+	}
+	s.chunks[s.circuits-1].length += s.total % int64(s.circuits)
+
+	// spawn initial fetchers
+	go s.progress()
+	for id := 0; id < s.circuits; id++ {
+		go s.fetchChunk(id)
+		time.Sleep(499 * time.Millisecond) // be gentle to the local tor daemon
+	}
+
+	// spawn additional fetchers as needed
+	for {
+		id := <-s.done
+		if s.chunks[id].length > 0 { // error
+			// resume in a new and hopefully faster circuit
+			s.chunks[id].circuit = seq
+			seq++
+			// fmt.Println("resume", s.chunks[id].length)
+		} else { // completed
+			longest := 0
+			s.mutex.Lock()
+			for i := 1; i < s.circuits; i++ {
+				if s.chunks[i].length > s.chunks[longest].length {
+					longest = i
+				}
+			}
+			s.mutex.Unlock()
+			// fmt.Println("completed", s.chunks[longest].length)
+			if s.chunks[longest].length == 0 { // all done
+				break
+			}
+			if s.chunks[longest].length <= 5*torBlock { // too short to split
+				continue
+			}
+			// this circuit is faster, so we split 80%/20%
+			s.mutex.Lock()
+			s.chunks[id].length = s.chunks[longest].length * 4 / 5
+			s.chunks[longest].length -= s.chunks[id].length
+			s.chunks[id].start = s.chunks[longest].start + s.chunks[longest].length
+			s.mutex.Unlock()
+		}
+		go s.fetchChunk(id)
+	}
+	return 0
+}
+
+func main() {
+	circuits := flag.Int("c", 20, "concurrent circuits")
+	flag.Usage = func() {
+		fmt.Fprintln(os.Stderr, "torget 1.0, a fast large file downloader over locally installed Tor")
+		fmt.Fprintln(os.Stderr, "Copyright © 2021 Michał Trojnara <[email protected]>")
+		fmt.Fprintln(os.Stderr, "Licensed under GNU/GPL version 3")
+		fmt.Fprintln(os.Stderr)
+		fmt.Fprintln(os.Stderr, "Usage: torget [-c circuits] url")
+		flag.PrintDefaults()
+	}
+	flag.Parse()
+	if flag.NArg() != 1 {
+		flag.Usage()
+		os.Exit(1)
+	}
+	state := NewState(*circuits)
+	os.Exit(state.Fetch(flag.Arg(0)))
+}
+
+// vim: noet:ts=4:sw=4:sts=4:spell