Skip to content

Commit

Permalink
Merge branch 'hotfix/0.10.2'
Browse files Browse the repository at this point in the history
  • Loading branch information
sutra committed Apr 23, 2024
2 parents 2994716 + 0dbfaf7 commit 0dc852e
Show file tree
Hide file tree
Showing 9 changed files with 8 additions and 19 deletions.
2 changes: 1 addition & 1 deletion pom.xml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<groupId>us.codecraft</groupId>
<version>0.10.1</version>
<version>0.10.2</version>
<modelVersion>4.0.0</modelVersion>
<packaging>pom</packaging>
<properties>
Expand Down
2 changes: 1 addition & 1 deletion webmagic-core/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
<parent>
<groupId>us.codecraft</groupId>
<artifactId>webmagic-parent</artifactId>
<version>0.10.1</version>
<version>0.10.2</version>
</parent>
<modelVersion>4.0.0</modelVersion>

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,6 @@
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request;
Expand All @@ -33,8 +31,6 @@
*/
public class HttpClientDownloader extends AbstractDownloader {

private Logger logger = LoggerFactory.getLogger(getClass());

private final Map<String, CloseableHttpClient> httpClients = new HashMap<String, CloseableHttpClient>();

private HttpClientGenerator httpClientGenerator = new HttpClientGenerator();
Expand Down Expand Up @@ -84,16 +80,10 @@ public Page download(Request request, Task task) {
try {
httpResponse = httpClient.execute(requestContext.getHttpUriRequest(), requestContext.getHttpClientContext());
page = handleResponse(request, request.getCharset() != null ? request.getCharset() : task.getSite().getCharset(), httpResponse, task);

onSuccess(page, task);
logger.info("Download page success: {}", request.getUrl());

return page;
} catch (IOException e) {

onError(page, task, e);
logger.info("Download page error: {}", request.getUrl(), e);

return page;
} finally {
if (httpResponse != null) {
Expand Down Expand Up @@ -138,7 +128,6 @@ private String getHtmlCharset(String contentType, byte[] contentBytes, Task task
String charset = CharsetUtils.detectCharset(contentType, contentBytes);
if (charset == null) {
charset = Optional.ofNullable(task.getSite().getDefaultCharset()).orElseGet(Charset.defaultCharset()::name);
logger.info("Charset autodetect failed, use {} as charset.", task.getSite().getDefaultCharset());
}
return charset;
}
Expand Down
2 changes: 1 addition & 1 deletion webmagic-coverage/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
<parent>
<groupId>us.codecraft</groupId>
<artifactId>webmagic-parent</artifactId>
<version>0.10.1</version>
<version>0.10.2</version>
</parent>

<artifactId>webmagic-coverage</artifactId>
Expand Down
2 changes: 1 addition & 1 deletion webmagic-extension/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
<parent>
<groupId>us.codecraft</groupId>
<artifactId>webmagic-parent</artifactId>
<version>0.10.1</version>
<version>0.10.2</version>
</parent>
<modelVersion>4.0.0</modelVersion>

Expand Down
2 changes: 1 addition & 1 deletion webmagic-samples/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
<parent>
<artifactId>webmagic-parent</artifactId>
<groupId>us.codecraft</groupId>
<version>0.10.1</version>
<version>0.10.2</version>
</parent>
<modelVersion>4.0.0</modelVersion>

Expand Down
2 changes: 1 addition & 1 deletion webmagic-saxon/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
<parent>
<artifactId>webmagic-parent</artifactId>
<groupId>us.codecraft</groupId>
<version>0.10.1</version>
<version>0.10.2</version>
</parent>
<modelVersion>4.0.0</modelVersion>

Expand Down
2 changes: 1 addition & 1 deletion webmagic-scripts/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
<parent>
<artifactId>webmagic-parent</artifactId>
<groupId>us.codecraft</groupId>
<version>0.10.1</version>
<version>0.10.2</version>
</parent>
<modelVersion>4.0.0</modelVersion>

Expand Down
2 changes: 1 addition & 1 deletion webmagic-selenium/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
<parent>
<artifactId>webmagic-parent</artifactId>
<groupId>us.codecraft</groupId>
<version>0.10.1</version>
<version>0.10.2</version>
</parent>
<modelVersion>4.0.0</modelVersion>

Expand Down

0 comments on commit 0dc852e

Please sign in to comment.