feat: add tianlang book support.

cmd/flags/config.go

@@ -45,6 +45,10 @@ var (
  ReLogin = false
  AppID = int64(0)
  AppHash = ""
+
+ // Tianlang secret key.
+
+ TianlangSecretKey = "359198"
 )
 
 // NewFetcher will create the fetcher by the command line arguments.

cmd/root.go

@@ -30,11 +30,15 @@ func Execute() {
 }
 
 func init() {
+ // Download commands.
  rootCmd.AddCommand(talebookCmd)
  rootCmd.AddCommand(sanqiuCmd)
+ rootCmd.AddCommand(tianlangCmd)
  rootCmd.AddCommand(telegramCmd)
- rootCmd.AddCommand(versionCmd)
+
+ // Tool commands.
  rootCmd.AddCommand(aliyunCmd)
+ rootCmd.AddCommand(versionCmd)
 
  persistentFlags := rootCmd.PersistentFlags()
 

cmd/sanqiu.go

@@ -9,8 +9,8 @@ import (
 )
 
 const (
- lowestBookID  = 163
- sanqiuWebsite = "https://www.sanqiu.mobi"
+ lowestSanqiuBookID = 163
+ sanqiuWebsite  = "https://www.sanqiu.mobi"
 )
 
 // sanqiuCmd used for download books from sanqiu.mobi
@@ -19,8 +19,8 @@ var sanqiuCmd = &cobra.Command{
  Short: "A tool for downloading books from sanqiu.mobi",
  Run: func(cmd *cobra.Command, args []string) {
  // Set the default start index.
- if flags.InitialBookID < lowestBookID {
- flags.InitialBookID = lowestBookID
+ if flags.InitialBookID < lowestSanqiuBookID {
+ flags.InitialBookID = lowestSanqiuBookID
  }
 
  // Print download configuration.

cmd/tianlang.go

@@ -0,0 +1,84 @@
+package cmd
+
+import (
+ "github.com/spf13/cobra"
+
+ "github.com/bookstairs/bookhunter/cmd/flags"
+ "github.com/bookstairs/bookhunter/internal/fetcher"
+ "github.com/bookstairs/bookhunter/internal/log"
+)
+
+const (
+ lowestTianlangBookID = 61
+ tianlangWebsite = "https://www.tianlangbooks.com"
+)
+
+var tianlangCmd = &cobra.Command{
+ Use: "tianlang",
+ Short: "A tool for downloading books from tianlangbooks.com",
+ Run: func(cmd *cobra.Command, args []string) {
+ if flags.InitialBookID < lowestTianlangBookID {
+ flags.InitialBookID = lowestTianlangBookID
+ }
+
+ // Print download configuration.
+ log.NewPrinter().
+ Title("Tianlang Download Information").
+ Head(log.DefaultHead...).
+ Row("Config Path", flags.ConfigRoot).
+ Row("Proxy", flags.Proxy).
+ Row("UserAgent", flags.UserAgent).
+ Row("Formats", flags.Formats).
+ Row("Extract Archive", flags.Extract).
+ Row("Download Path", flags.DownloadPath).
+ Row("Initial ID", flags.InitialBookID).
+ Row("Rename File", flags.Rename).
+ Row("Thread", flags.Thread).
+ Row("Request Per Minute", flags.RateLimit).
+ Row("Secret key", flags.TianlangSecretKey).
+ Row("Aliyun RefreshToken", flags.HideSensitive(flags.RefreshToken)).
+ Row("Telecom Username", flags.HideSensitive(flags.Username)).
+ Row("Telecom Password", flags.HideSensitive(flags.Password)).
+ Print()
+
+ // Set the domain for using in the client.Client.
+ flags.Website = tianlangWebsite
+
+ // Create the fetcher.
+ properties := flags.NewDriverProperties()
+ properties["secretKey"] = flags.TianlangSecretKey
+ f, err := flags.NewFetcher(fetcher.TianLang, properties)
+ log.Exit(err)
+
+ // Wait all the threads have finished.
+ err = f.Download()
+ log.Exit(err)
+
+ // Finished all the tasks.
+ log.Info("Successfully download all the books.")
+ },
+}
+
+func init() {
+ f := tianlangCmd.Flags()
+
+ // Common download flags.
+ f.StringSliceVarP(&flags.Formats, "format", "f", flags.Formats, "The file formats you want to download")
+ f.BoolVarP(&flags.Extract, "extract", "e", flags.Extract, "Extract the archive file for filtering")
+ f.StringVarP(&flags.DownloadPath, "download", "d", flags.DownloadPath, "The book directory you want to use")
+ f.Int64VarP(&flags.InitialBookID, "initial", "i", flags.InitialBookID, "The book id you want to start download")
+ f.BoolVarP(&flags.Rename, "rename", "r", flags.Rename, "Rename the book file by book id")
+ f.IntVarP(&flags.Thread, "thread", "t", flags.Thread, "The number of download thead")
+ f.IntVar(&flags.RateLimit, "ratelimit", flags.RateLimit, "The allowed requests per minutes")
+
+ // Tianlang books flags.
+ f.StringVar(&flags.TianlangSecretKey, "secretKey", flags.TianlangSecretKey, "The secret key for tianlang")
+
+ // Drive ISP flags.
+ f.StringVar(&flags.Driver, "source", flags.Driver, "The source (aliyun, telecom, lanzou) to download book")
+ f.StringVar(&flags.RefreshToken, "refreshToken", flags.RefreshToken, "Refresh token for aliyun drive")
+ f.StringVar(&flags.TelecomUsername, "telecomUsername", flags.TelecomUsername, "Telecom drive username")
+ f.StringVar(&flags.TelecomPassword, "telecomPassword", flags.TelecomPassword, "Telecom drive password")
+
+ _ = tianlangCmd.MarkFlagRequired("source")
+}

internal/driver/lanzou.go

@@ -1,7 +1,6 @@
 package driver
 
 import (
- "errors"
  "io"
 
  "github.com/bookstairs/bookhunter/internal/client"
@@ -14,7 +13,7 @@ func newLanzouDriver(c *client.Config, _ map[string]string) (Driver, error) {
  return nil, err
  }
 
- return &lanzouDriver{driver: drive}, errors.New("we don't support lanzou currently")
+ return &lanzouDriver{driver: drive}, nil
 }
 
 type lanzouDriver struct {

internal/fetcher/sanqiu.go

@@ -1,130 +1,66 @@
 package fetcher
 
 import (
- "errors"
- "io"
  "strconv"
+ "strings"
+
+ "github.com/PuerkitoBio/goquery"
 
  "github.com/bookstairs/bookhunter/internal/client"
  "github.com/bookstairs/bookhunter/internal/driver"
- "github.com/bookstairs/bookhunter/internal/file"
- "github.com/bookstairs/bookhunter/internal/log"
- "github.com/bookstairs/bookhunter/internal/naming"
- "github.com/bookstairs/bookhunter/internal/sanqiu"
-)
-
-var (
- ErrEmptySanqiu = errors.New("couldn't find available books in sanqiu")
+ "github.com/bookstairs/bookhunter/internal/wordpress"
 )
 
-type sanqiuService struct {
- config *Config
- client *client.Client
- driver driver.Driver
-}
-
 func newSanqiuService(config *Config) (service, error) {
- // Create the resty client for HTTP handing.
- c, err := client.New(config.Config)
- if err != nil {
- return nil, err
- }
-
- // Create the net disk driver.
- d, err := driver.New(config.Config, config.Properties)
- if err != nil {
- return nil, err
- }
-
- return &sanqiuService{
- config: config,
- client: c,
- driver: d,
- }, nil
-}
-
-func (s *sanqiuService) size() (int64, error) {
- resp, err := s.client.R().
- SetQueryParams(map[string]string{
- "orderby": "id",
- "order": "desc",
- "per_page": "1",
- }).
- Get("/wp-json/wp/v2/posts")
-
- if err != nil {
- return 0, err
- }
-
- books := make([]sanqiu.BookResp, 0, 1)
- err = sanqiu.ParseAPIResponse(resp, &books)
- if err != nil {
- return 0, err
- }
-
- if len(books) < 1 {
- return 0, ErrEmptySanqiu
- }
-
- return books[0].ID, nil
-}
-
-func (s *sanqiuService) formats(id int64) (map[Format]driver.Share, error) {
- resp, err := s.client.R().
- SetQueryParam("id", strconv.FormatInt(id, 10)).
- Get("/download.php")
- if err != nil {
- return nil, err
- }
-
- links, err := sanqiu.DownloadLinks(resp.String())
- if err != nil {
- return nil, err
- }
-
- for source, link := range links {
- if source != s.driver.Source() {
- continue
+ return newWordpressService(config, func(c *client.Client, id int64) (map[driver.Source]wordpress.ShareLink, error) {
+ resp, err := c.R().
+ SetQueryParam("id", strconv.FormatInt(id, 10)).
+ Get("/download.php")
+ if err != nil {
+ return nil, err
  }
 
- shares, err := s.driver.Resolve(link.URL, link.Code)
+ doc, err := goquery.NewDocumentFromReader(strings.NewReader(resp.String()))
  if err != nil {
  return nil, err
  }
 
- res := make(map[Format]driver.Share)
- for _, share := range shares {
- if ext, has := naming.Extension(share.FileName); has {
- if format, err := ParseFormat(ext); err == nil {
- res[format] = share
- } else {
- log.Debugf("The file name %s don't have valid extension %s", share.FileName, ext)
+ // Find all the links.
+ links := map[driver.Source]wordpress.ShareLink{}
+ doc.Find(".downfile a").Each(func(i int, selection *goquery.Selection) {
+ driveName := selection.Text()
+ href, exists := selection.Attr("href")
+ if exists {
+ for linkType, name := range driveNamings {
+ if strings.Contains(driveName, name) {
+ links[linkType] = wordpress.ShareLink{URL: href}
+ break
+ }
  }
- } else {
- log.Debugf("The file name %s don't have the extension", share.FileName)
  }
- }
+ })
 
- return res, nil
- }
-
- log.Debug("No downloadable files found in this link.")
- return map[Format]driver.Share{}, nil
-}
-
-func (s *sanqiuService) fetch(_ int64, _ Format, share driver.Share, writer file.Writer) error {
- content, size, err := s.driver.Download(share)
- if err != nil {
- return err
- }
- defer func() { _ = content.Close() }()
+ if len(links) == 0 {
+ return map[driver.Source]wordpress.ShareLink{}, nil
+ }
 
- // Set download progress if required.
- if size > 0 {
- writer.SetSize(size)
- }
+ // Find all the passcodes.
+ doc.Find(".plus_l li").Each(func(i int, selection *goquery.Selection) {
+ text := selection.Text()
+ for linkType, link := range links {
+ name := driveNamings[linkType]
+ if strings.Contains(text, name) {
+ match := sanqiuPasscodeRe.FindStringSubmatch(text)
+ if len(match) == 2 {
+ links[linkType] = wordpress.ShareLink{
+ URL: link.URL,
+ Code: match[1],
+ }
+ }
+ }
+ }
+ })
 
- // Save the download content info files.
- _, err = io.Copy(writer, content)
- return err
+ return links, nil
+ })
 }

internal/fetcher/service.go

@@ -27,12 +27,10 @@ func newService(c *Config) (service, error) {
  return newTalebookService(c)
  case SanQiu:
  return newSanqiuService(c)
- case SoBooks:
- // TODO We are working on this feature now.
- return nil, errors.New("we don't support sobooks now")
  case TianLang:
- // TODO We are working on this feature now.
- return nil, errors.New("we don't support tianlang now")
+ return newTianlangService(c)
+ case SoBooks:
+ return nil, errors.New("TODO we don't support sobooks now")
  default:
  return nil, fmt.Errorf("no such fetcher service [%s] supported", c.Category)
  }