pref: add status to site data

Made improvement
- add existRegex, avoid wrong result as much as possible
- fix 52pojie site data
- add status to site data
- fix nsfw options
- add no site data information
- fix wikipedia site data
- add site data

cmd/detect.go

@@ -28,14 +28,17 @@ type detectArgsType struct {
 }
 
 var (
-	detectArgs   detectArgsType
-	wg           sync.WaitGroup
-	existInfo    = "[+] %-15s %-15s: %s\n"
-	nonExistInfo = "[-] %-15s %-15s: non exists\n"
-	reqErrorInfo = "[!] %-15s %-15s: %s requests error, retry %d/%d\n"
-	sleepMap     = make(map[string]int64)
-	sleepChannel = make(map[string]chan bool)
-	searchInfo   = "[+] %-15s %-15s: Please check in %s\n"
+	detectArgs      detectArgsType
+	wg              sync.WaitGroup
+	nonSiteData     = "[-] There is no site data of %s\n"
+	existInfo       = "[+] %-15s %-15s: %s\n"
+	nonExistInfo    = "[-] %-15s %-15s: non exists\n"
+	reqErrorInfo    = "[!] %-15s %-15s: %s requests error, retry %d/%d\n"
+	sleepMap        = make(map[string]int64)
+	sleepChannel    = make(map[string]chan bool)
+	searchInfo      = "[+] %-15s %-15s: Please check in %s\n"
+	disableSiteInfo = "[!] %-15s %-15s: data of %s is temporarily unavailable\n"
+	nsfwInfo        = "[!] %-15s %-15s: %s is nsfw\n"
 )
 
 func init() {
@@ -80,7 +83,11 @@ func detect(_ *cobra.Command, _ []string) {
 		log.Infoln("Detect to ", detectArgs.site)
 		siteDataMap = make(map[string]gjson.Result)
 		for _, site := range detectArgs.site {
-			siteDataMap[site] = r.Get(strings.ToLower(site))
+			if siteData := r.Get(strings.ToLower(site)); siteData.Exists() {
+				siteDataMap[site] = r.Get(strings.ToLower(site))
+			} else {
+				log.Infof(nonSiteData, site)
+			}
 		}
 	}
 
@@ -113,7 +120,8 @@ func detectSite(name, site string, siteBody gjson.Result) {
 	// flag for precisely mode
 	flag := false
 
-	if !siteBody.Get("isNFSW").Bool() && detectArgs.isNSFW {
+	if siteBody.Get("isNFSW").Bool() && !detectArgs.isNSFW {
+		log.Debugf(nsfwInfo, name, site, site)
 		return
 	}
 
@@ -134,6 +142,11 @@ func detectSite(name, site string, siteBody gjson.Result) {
 
 	detectCount := len(detectReq) - 1
 	for index, detectData := range detectReq {
+
+		if status := detectData.Get("status"); status.Exists() && !status.Bool() {
+			log.Debugf(disableSiteInfo, name, site, site)
+			break
+		}
 		retryTimes := 0
 		if detectUser(name, site, index, retryTimes, detectCount, &flag, detectData) {
 			continue
@@ -209,7 +222,7 @@ func detectUser(name, site string, requestTimes, retryTimes, detectCount int, fl
 	}
 
 	if retryTimes == detectArgs.retry {
-		log.Infof(reqErrorInfo, name, site, url, retryTimes, detectArgs.retry)
+		//log.Infof(reqErrorInfo, name, site, url, retryTimes, detectArgs.retry)
 		return false
 	}
 
@@ -219,7 +232,7 @@ func detectUser(name, site string, requestTimes, retryTimes, detectCount int, fl
 	//
 	//log.Debugln(rep.Request.Header)
 	//
-	//log.Debugln(rep.String())
+	//log.Debugln(rep.Status(), rep.String())
 
 	// statusCode, existRegex must both be true
 	statusCode := detectData.Get("statusCode")

cmd/gen.go

@@ -11,6 +11,7 @@ import (
 
 type genArgsType struct {
 	fileName string
+	data     string
 	//proxy    string
 }
 
@@ -20,6 +21,7 @@ var (
 
 func init() {
 	genCmd.Flags().StringVarP(&genArgs.fileName, "filename", "f", "site.md", "Markdown filename to generate")
+	genCmd.Flags().StringVarP(&genArgs.data, "data", "d", "data.json", "Site data file")
 	//genCmd.Flags().StringVarP(&genArgs.proxy, "proxy", "p", "", "Make requests over a proxy. e.g. socks5://127.0.0.1:1080")
 	rootCmd.AddCommand(genCmd)
 }
@@ -37,7 +39,7 @@ func gen(_ *cobra.Command, _ []string) {
 		log.SetLevel(log.DebugLevel)
 	}
 
-	siteData, err := ioutil.ReadFile("data.json")
+	siteData, err := ioutil.ReadFile(genArgs.data)
 	if err != nil {
 		log.Fatalln(err)
 	}