Upgrade build and dependencies

.github/workflows/build.yml

@@ -0,0 +1,29 @@
+name: cc-webgraph build
+
+on:
+ push:
+ branches:
+ - main
+ pull_request:
+ branches:
+ - main
+
+jobs:
+ build:
+ runs-on: ubuntu-latest
+ strategy:
+ matrix:
+ java: [ 11, 17, 21 ]
+ name: Java ${{ matrix.java }}
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Setup JDK
+ uses: actions/setup-java@v4
+ with:
+ distribution: 'temurin'
+ java-version: ${{ matrix.java }}
+ cache: 'maven'
+
+ - name: Build
+ run: mvn verify javadoc:aggregate

README.md

@@ -4,11 +4,14 @@ Tools to construct and process web graphs from Common Crawl data
 
 ## Compiling and Packaging Java Tools
 
+Java 11 or upwards are required.
+
 The Java tools are compiled and packaged by [Maven](https://maven.apache.org/). If Maven is installed just run `mvn package`. Now the Java tools can be run via
 ```
 java -cp target/cc-webgraph-0.1-SNAPSHOT-jar-with-dependencies.jar <classname> <args>...
 ```
-The assembly jar file requires Java 10 or upwards to run. It includes also the [WebGraph](https://webgraph.di.unimi.it/) and [LAW](https://law.di.unimi.it/software.php) packages required to compute [PageRank](https://en.wikipedia.org/wiki/PageRank) and [Harmonic Centrality](https://en.wikipedia.org/wiki/Centrality#Harmonic_centrality).
+
+The assembly jar file includes also the [WebGraph](https://webgraph.di.unimi.it/) and [LAW](https://law.di.unimi.it/software.php) packages required to compute [PageRank](https://en.wikipedia.org/wiki/PageRank) and [Harmonic Centrality](https://en.wikipedia.org/wiki/Centrality#Harmonic_centrality).
 
 Note that the webgraphs are usually multiple Gigabytes in size and require a sufficient Java heap size ([Java option](https://docs.oracle.com/en/java/javase/14/docs/specs/man/java.html#extra-options-for-java) `-Xmx`) for processing.
 

pom.xml

@@ -12,17 +12,17 @@
 
  <properties>
  <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
- <java.version>1.8</java.version>
+ <java.version>11</java.version>
 
  <webgraph.version>3.6.10</webgraph.version>
  <webgraph.big.version>3.7.0</webgraph.big.version>
  <law.version>2.7.2</law.version>
- <fastutil.version>8.5.12</fastutil.version>
+ <fastutil.version>8.5.13</fastutil.version>
  <crawler.commons.version>1.4</crawler.commons.version>
 
- <slf4j-api.version>2.0.7</slf4j-api.version>
+ <slf4j-api.version>2.0.12</slf4j-api.version>
 
- <junit.version>5.10.0</junit.version>
+ <junit.version>5.10.2</junit.version>
  </properties>
 
  <build>
@@ -34,15 +34,15 @@
  <plugins>
  <plugin>
  <artifactId>maven-compiler-plugin</artifactId>
- <version>3.8.0</version>
+ <version>3.13.0</version>
  <configuration>
  <source>${java.version}</source>
  <target>${java.version}</target>
  </configuration>
  </plugin>
  <plugin>
  <artifactId>maven-assembly-plugin</artifactId>
- <version>3.1.1</version>
+ <version>3.7.1</version>
  <configuration>
  <descriptorRefs>
  <descriptorRef>jar-with-dependencies</descriptorRef>
@@ -60,7 +60,7 @@
  </plugin>
  <plugin>
  <artifactId>maven-surefire-plugin</artifactId>
- <version>3.0.0-M6</version>
+ <version>3.2.5</version>
  </plugin>
  </plugins>
  </build>
@@ -145,10 +145,30 @@
  <artifactId>law</artifactId>
  <version>${law.version}</version>
  <exclusions>
+ <exclusion>
+ <groupId>net.sf.jung</groupId>
+ <artifactId>jung-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>net.sf.jung</groupId>
+ <artifactId>jung-io</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpasyncclient</artifactId>
+ </exclusion>
  <exclusion>
  <groupId>org.eclipse.jetty.aggregate</groupId>
  <artifactId>jetty-all</artifactId>
  </exclusion>
+ <exclusion>
+ <groupId>org.softee</groupId>
+ <artifactId>pojo-mbean</artifactId>
+ </exclusion>
  <exclusion>
  <groupId>com.fasterxml.jackson</groupId>
  <artifactId>jackson-bom</artifactId>

src/main/java/org/commoncrawl/webgraph/HostToDomainGraph.java

@@ -31,9 +31,9 @@
  * represented by two text files/streams with tab-separated columns
  * <dl>
  * <dt>vertices</dt>
- * <dd>&langle;id, revName&rangle;</dd>
+ * <dd>&lt;id, revName&gt;</dd>
  * <dt>edges</dt>
- * <dd>&langle;fromId, toId&rangle;</dd>
+ * <dd>&lt;fromId, toId&gt;</dd>
  * </dl>
  * Host or domain names are reversed (<code>www.example.com</code> is written as
  * <code>com.example.www</code>). The vertices file is sorted lexicographically
@@ -47,19 +47,20 @@
  * 
  * <p>
  * Notes, assumptions and preconditions:
+ * </p>
  * <ul>
- * <li>host nodes must be sorted lexicographically by reversed host name, see
+ * <li>host vertices must be sorted lexicographically by reversed host name, see
  * above</li>
  * <li>the host-domain map is hold as array. To overcome Java's max array size
  * (approx. 2^32 or {@link Arrays#MAX_ARRAY_SIZE}) {@link HostToDomainGraphBig}
- * (based on fastutils' {@link BigArrays}) is automatically used if the array
- * size limit is hit.</li>
+ * (based on fastutils' {@link BigArrays}) is used if the array size limit is
+ * hit by the number of hosts. This number (or an estimate) needs to be known
+ * ahead.</li>
  * <li>the number of resulting domains is limited by Java's max. array size.
  * This shouldn't be a problem.</li>
  * <li>also the number of hosts per domain is limited by Java's max. array
- * size</li>
+ * size.</li>
  * </ul>
- * </p>
  */
 public class HostToDomainGraph {
 
@@ -160,6 +161,11 @@ public int compareTo(Domain o) {
  /**
  * Whether the domain is safe to output given the reversed domain name seen
  * next.
+ * 
+ * @param nextDomainRevName next name in lexicographically sorted list of
+ * reversed domain names
+ * @return true if the domain is safe to output, that is from a list of sorted
+ * host names no host later in this list may fold to this domain name
  */
  public boolean isSafeToOutput(String nextDomainRevName) {
  return isSafeToOutput(this.revName, nextDomainRevName);
@@ -273,7 +279,7 @@ public void multiPartSuffixesAsDomains(boolean include) {
 
  /**
  * Reverse host name, eg. <code>www.example.com</code> is reversed to
- * <code>com.example.www</code>. Can be also used to "unreverse" a reversed host
+ * <code>com.example.www</code>. Can also be used to "unreverse" a reversed host
  * name.
  * 
  * @param host name
@@ -462,10 +468,12 @@ public HostToDomainGraphBig(long maxSize) {
  ids = LongBigArrays.newBigArray(maxSize);
  }
 
+ @Override
  protected void setValue(long id, long value) {
  BigArrays.set(ids, id, value);
  }
 
+ @Override
  protected long getValue(long id) {
  return BigArrays.get(ids, id);
  }

src/script/host2domaingraph.sh

@@ -28,8 +28,8 @@ if [ $# -lt 3 ]; then
  if [ ${#FLAGS[@]} -gt 0 ]; then
  echo ""
  echo "Calling HostToDomainGraph with provided flags (${FLAGS[*]}):"
- "$JAVA_HOME"/bin/java -cp "$CLASSPATH":"JAR" \
-  "${PROPERTIES[@]}" org.commoncrawl.webgraph.HostToDomainGraph "${FLAGS[@]}"
+ "$JAVA_HOME"/bin/java -cp "$CLASSPATH":"$JAR" "${PROPERTIES[@]}" \
+ org.commoncrawl.webgraph.HostToDomainGraph "${FLAGS[@]}"
  fi
  exit 1
 fi