Merge pull request #64 from bit4woo/dev20240429

Dev20240429 merge to master

pom.xml

@@ -33,6 +33,7 @@
 						<!-- 编译时排除Deprecated包及下面的资源 -->
 						<exclude>**/Deprecated/**</exclude>
 						<exclude>**/test/**</exclude>
+						<exclude>**/toElastic/**</exclude>
 					</excludes>
 				</configuration>
 			</plugin>
@@ -93,10 +94,11 @@
 			<artifactId>burp-extender-api</artifactId>
 			<version>[1.7.22,)</version>
 		</dependency>
+
 		<dependency>
 			<groupId>com.github.bit4woo</groupId>
-			<artifactId>burp-api-common</artifactId>
-			<version>master-SNAPSHOT</version>
+			<artifactId>java_utilbox</artifactId>
+			<version>main-SNAPSHOT</version>
 		</dependency>
 		<!-- https://mvnrepository.com/artifact/com.alibaba/fastjson -->
 		<dependency>
@@ -136,7 +138,7 @@
 		<dependency>
 			<groupId>org.xerial</groupId>
 			<artifactId>sqlite-jdbc</artifactId>
-			<version>3.36.0.1</version>
+			<version>3.41.2.2</version>
 		</dependency>
 		<!-- https://mvnrepository.com/artifact/commons-net/commons-net -->
 		<dependency>
@@ -176,35 +178,35 @@
 			<version>2.0b6</version>
 		</dependency>
 		<!-- Elasticsearch High-Level REST Client -->
-		<dependency>
+		<!--<dependency>
 			<groupId>org.elasticsearch.client</groupId>
 			<artifactId>elasticsearch-rest-high-level-client</artifactId>
 			<version>7.15.0</version>
-		</dependency>
+		</dependency>-->
 		<!-- Elasticsearch Low-Level REST Client -->
-		<dependency>
+		<!--<dependency>
 			<groupId>org.elasticsearch.client</groupId>
 			<artifactId>elasticsearch-rest-client</artifactId>
-			<version>7.15.0</version>
-		</dependency>
+			<version>7.17.14</version>
+		</dependency>-->
 		<!-- Elasticsearch Core -->
-		<dependency>
+		<!--<dependency>
 			<groupId>org.elasticsearch</groupId>
 			<artifactId>elasticsearch</artifactId>
-			<version>7.15.0</version>
+			<version>7.17.14</version>
 			<exclusions>
 				<exclusion>
 					<groupId>*</groupId>
 					<artifactId>*</artifactId>
 				</exclusion>
 			</exclusions>
-		</dependency>
+		</dependency>-->
 		<!-- Elasticsearch XContent -->
-		<dependency>
+		<!--<dependency>
 			<groupId>org.elasticsearch</groupId>
 			<artifactId>elasticsearch-x-content</artifactId>
-			<version>7.15.0</version>
-		</dependency>
+			<version>7.17.14</version>
+		</dependency>-->
 		<dependency>
 			<groupId>org.apache.lucene</groupId>
 			<artifactId>lucene-core</artifactId>

src/ASN/ASNEntry.java

@@ -4,7 +4,7 @@
 import inet.ipaddr.IPAddress;
 import inet.ipaddr.IPAddressSeqRange;
 import inet.ipaddr.IPAddressString;
-import utils.IPAddressUtils;
+import com.bit4woo.utilbox.utils.IPAddressUtils;
 import org.apache.commons.lang3.StringUtils;
 import java.util.List;
 
@@ -139,7 +139,7 @@ public boolean contains(String IP){
 			try {
 				String start = prefix.split("-")[0];
 				String end = prefix.split("-")[1];
-				return IPAddressUtils.checkIPIsInGivenRange(IP,start,end);//
+				return IPAddressUtils.IsInRange(IP,start,end);//
 			} catch (AddressStringException e) {
 				e.printStackTrace();
 				return false;

src/ASN/ASNQuery.java

@@ -17,7 +17,7 @@
 import com.github.kevinsawicki.http.HttpRequest;
 
 import burp.BurpExtender;
-import utils.IPAddressUtils;
+import com.bit4woo.utilbox.utils.IPAddressUtils;
 
 public class ASNQuery {
 	public static final String localdir =
@@ -200,7 +200,7 @@ private static void batchQueryFromApi(List<String> ipSet){
 	 * @return ASNEntry{asn='3215', asname_long='AS3215', asname_short='', prefix='2.0.0.0-2.15.255.255', geo='FR'}
 	 */
 	public ASNEntry query(String singleIP){
-		if (IPAddressUtils.isValidIP(singleIP)){
+		if (IPAddressUtils.isValidIPv4NoPort(singleIP)){
 			//1.从缓存查询
 			ASNEntry result = queryFromRecent(singleIP);
 			//2.从本地数据库文件查询

src/Deprecated/DirBruterProducer.java

@@ -71,7 +71,7 @@ public void run() {
 				IHttpRequestResponse messageinfo = callbacks.makeHttpRequest(service, request);
 				int leftTaskNum = pathDict.size();
 				stdout.println(String.format("%s tasks left, Runner Checking: %s",leftTaskNum,url.toString()));
-				Getter getter = new Getter(helpers);
+				HelperPlus getter = BurpExtender.getHelperPlus();
 				if (messageinfo !=null) {
 					byte[] response = messageinfo.getResponse();
 					int status = getter.getStatusCode(messageinfo);

src/Deprecated/GatewayBypassChecker.java

@@ -28,7 +28,7 @@ public class GatewayBypassChecker extends Thread {//Producer do
 	public PrintWriter stdout = new PrintWriter(callbacks.getStdout(), true);
 	public PrintWriter stderr = new PrintWriter(callbacks.getStderr(), true);
 	public IExtensionHelpers helpers = callbacks.getHelpers();
-	public HelperPlus getter = new HelperPlus(helpers);
+	public HelperPlus getter = BurpExtender.getHelperPlus();
 
 	LineTableModel runnerTableModel;
 

src/Deprecated/RootDomainForBrute.java

@@ -71,7 +71,7 @@ public HashMap<String,Set<String>> query(String domain){
 		HashMap<String,Set<String>> result = new HashMap();
 		for (String server:nameServers){
 			try {
-				result = DomainNameUtils.dnsquery(domain,server);
+				result = DomainUtils.dnsquery(domain,server);
 				return result;
 			}catch (Exception e){
 				;//do nothing

src/Deprecated/ThreadBruteDomain.java

@@ -63,7 +63,7 @@ public void Do(){
         stdout.println("checking wildcard DNS record");
         for (String rootDomain: rootDomains){
             String badDomain = "domain-hunter-pro-test."+rootDomain;
-            Set<String> ipset = DomainNameUtils.dnsquery(badDomain).get("IP");
+            Set<String> ipset = DomainUtils.dnsquery(badDomain).get("IP");
             badRecords.put(rootDomain,ipset);
         }
 

src/InternetSearch/APISearchAction.java

@@ -4,23 +4,31 @@
 import java.io.PrintWriter;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 import javax.swing.AbstractAction;
 import javax.swing.Action;
+import javax.swing.JOptionPane;
 import javax.swing.SwingWorker;
 import javax.swing.table.AbstractTableModel;
 
-import InternetSearch.Client.*;
 import org.apache.commons.lang3.StringUtils;
 
+import com.bit4woo.utilbox.utils.IPAddressUtils;
+
+import InternetSearch.Client.FoFaClient;
+import InternetSearch.Client.HunterClient;
+import InternetSearch.Client.HunterIoClient;
+import InternetSearch.Client.QuakeClient;
+import InternetSearch.Client.ShodanClient;
+import InternetSearch.Client.ZoomEyeClient;
 import burp.BurpExtender;
-import burp.IPAddressUtils;
 import domain.DomainManager;
 import domain.target.TargetTableModel;
 import title.LineTableModel;
-import utils.GrepUtils;
 
 public class APISearchAction extends AbstractAction {
 
@@ -84,8 +92,9 @@ public final void actionPerformed(ActionEvent e) {
         SwingWorker<Map, Map> worker = new SwingWorker<Map, Map>() {
             @Override
             protected Map doInBackground() throws Exception {
-
+            	Set<String> searchedContent = new HashSet<String>();
                 if (modelRows.length >= 50) {
+                	JOptionPane.showMessageDialog(null, "too many items selected!! should less than 50","Alert",JOptionPane.WARNING_MESSAGE);
                     stderr.print("too many items selected!! should less than 50");
                     return null;
                 }
@@ -112,8 +121,15 @@ protected Map doInBackground() throws Exception {
                         searchContent = result.second;
                     }
 
-                    DoSearchAllInOn(searchType, searchContent, APISearchAction.this.engineList);
-
+                    String tabname = String.format("%s(%s)", searchType, searchContent);
+                    if (searchedContent.add(tabname)) {
+                    	//保证单次操作，不对相同项进行重复搜索
+                    	Set<String> already = BurpExtender.getGui().getSearchPanel().getAlreadySearchContent();
+                    	if (!already.contains(tabname)) {
+                    		//保证已经存在的搜索内容不再重复
+                    		DoSearchAllInOn(searchType, searchContent, APISearchAction.this.engineList);
+                    	}
+                    }
                 }
                 return null;
             }
@@ -196,9 +212,9 @@ public static List<SearchResultEntry> DoSearchAllInOn(String searchType, String
                 String host = entry.getHost();
                 String rootDomain = entry.getRootDomain();
                 result.addIfValid(host);
-                List<String> ips = GrepUtils.grepIPAndPort(host);
+                List<String> ips = IPAddressUtils.grepIPv4MayPort(host);
                 for (String ip : ips) {
-                    if (IPAddressUtils.isValidIP(ip)) {
+                    if (IPAddressUtils.isValidIPv4MayPort(ip)) {
                         result.getSpecialPortTargets().add(ip);
                     }
                 }

src/InternetSearch/BrowserSearchAction.java

@@ -14,8 +14,9 @@
 import burp.BurpExtender;
 import domain.target.TargetTableModel;
 import title.LineTableModel;
-import utils.DomainNameUtils;
-import utils.IPAddressUtils;
+import com.bit4woo.utilbox.utils.DomainUtils;
+import com.bit4woo.utilbox.utils.IPAddressUtils;
+import com.bit4woo.utilbox.utils.SystemUtils;
 
 
 
@@ -80,7 +81,7 @@ public final void actionPerformed(ActionEvent e) {
 			String url = buildSearchUrl(engine,searchContent);
 
 			try {
-				Commons.browserOpen(url, null);
+				SystemUtils.browserOpen(url, null);
 			} catch (Exception err) {
 				err.printStackTrace(BurpExtender.getStderr());
 			}
@@ -130,13 +131,13 @@ else if (engine.equalsIgnoreCase(SearchEngine.ASN_INFO_BGP_HE_NET)) {
 			//https://bgp.he.net/dns/shopee.com
 			//https://bgp.he.net/net/143.92.111.0/24
 			//https://bgp.he.net/ip/143.92.127.1
-			if (IPAddressUtils.isValidIP(searchContent)){
+			if (IPAddressUtils.isValidIPv4NoPort(searchContent)){
 				url = "https://bgp.he.net/ip/"+searchContent;
 			}
 			if (IPAddressUtils.isValidSubnet(searchContent)){
 				url = "https://bgp.he.net/net/"+searchContent;
 			}
-			if (DomainNameUtils.isValidDomain(searchContent)){
+			if (DomainUtils.isValidDomainNoPort(searchContent)){
 				url = "https://bgp.he.net/dns/"+searchContent;
 			}
 		}

src/InternetSearch/Client/BaseClient.java

@@ -5,14 +5,14 @@
 import java.util.ArrayList;
 import java.util.List;
 
-import InternetSearch.SearchEngine;
-import InternetSearch.SearchType;
 import org.apache.commons.lang3.StringUtils;
 
+import com.bit4woo.utilbox.utils.UrlUtils;
+
 import InternetSearch.HttpClientOfBurp;
+import InternetSearch.SearchEngine;
 import InternetSearch.SearchResultEntry;
 import burp.BurpExtender;
-import utils.URLUtils;
 
 /**
  * 什么时候使用Interface，什么时候使用abstract class?
@@ -54,7 +54,7 @@ public List<String> Search(String searchContent, String searchType) {
                 String searchDork = buildSearchDork(searchContent, searchType);
                 this.url = buildSearchUrl(searchDork, page);
                 this.raw = buildRawData(searchDork, page);
-                if (URLUtils.isVaildUrl(url)) {
+                if (UrlUtils.isVaildUrl(url)) {
                     String body = HttpClientOfBurp.doRequest(new URL(url), raw);
                     this.resp_body = body;
                     if (body.length() <= 0) {

src/InternetSearch/Client/ZoomEyeClient.java

@@ -4,7 +4,9 @@
 import java.io.IOException;
 import java.net.URLEncoder;
 import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang3.StringUtils;
@@ -35,24 +37,15 @@ public List<SearchResultEntry> parseResp(String respbody) {
 				for (Object item : results) {
 
 					JSONObject entryitem = (JSONObject) item;
-
+					Set<String> ipSet = getIPSet(entryitem);
 					SearchResultEntry entry = new SearchResultEntry();
-					try {
-						//title:xxx 获得的是IP string
-						entry.getIPSet().add(entryitem.getString("ip"));
-					}catch(Exception e) {
-						//site:xxx.com 获得的是IP List
-						JSONArray ipList = entryitem.getJSONArray("ip");
-						for (int i = 0; i < ipList.length(); i++) {
-				            String element = ipList.getString(i);
-				            entry.getIPSet().add(element);
-				        }
-					}
-
+
+					entry.getIPSet().addAll(ipSet);
+
 					try {
 						entry.setHost(entryitem.getString("rdns"));
 					} catch (Exception e) {
-						entry.setHost(entryitem.getString("ip"));
+						entry.setHost((String)(ipSet.toArray())[0]);
 					}
 
 					int port = entryitem.getJSONObject("portinfo").getInt("port");
@@ -76,6 +69,23 @@ public List<SearchResultEntry> parseResp(String respbody) {
 		return result;
 	}
 
+	public static Set<String> getIPSet(JSONObject entryitem){
+		Set<String> result = new HashSet<String>();
+
+		try {
+			//title:xxx 获得的是IP string
+			result.add(entryitem.getString("ip"));
+		}catch(Exception e) {
+			//site:xxx.com 获得的是IP List
+			JSONArray ipList = entryitem.getJSONArray("ip");
+			for (int i = 0; i < ipList.length(); i++) {
+				String element = ipList.getString(i);
+				result.add(element);
+			}
+		}
+		return result;
+	}
+
 	@Override
 	public boolean hasNextPage(String respbody,int currentPage) {
 		// "size":83,"page":1,

src/InternetSearch/HttpClientOfBurp.java

@@ -3,9 +3,10 @@
 import java.net.URL;
 import java.util.Date;
 
+import com.bit4woo.utilbox.burp.HelperPlus;
+
 import base.Commons;
 import burp.BurpExtender;
-import burp.HelperPlus;
 import burp.IBurpExtenderCallbacks;
 import burp.IExtensionHelpers;
 import burp.IHttpRequestResponse;
@@ -56,7 +57,7 @@ public static String doRequest(URL url,byte[] byteRequest) {
 		IHttpService service =getHttpService(url);
 		IHttpRequestResponse message = callbacks.makeHttpRequest(service, byteRequest);
 
-		HelperPlus getter = new HelperPlus(helpers);
+		HelperPlus getter = BurpExtender.getHelperPlus();
 		int code = getter.getStatusCode(message);
 
 		if (ConfigManager.getBooleanConfigByKey(ConfigName.ApiReqToTitle)