Merge branch 'main' into patch-3

allenai · Sep 20, 2024 · ec0429e · ec0429e
2 parents 5f77f03 + 621a6f4
commit ec0429e
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 3 deletions.
diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml
@@ -279,7 +279,7 @@ jobs:
     if: "startsWith(github.ref, 'refs/tags/')"
     needs: [build-linux, build-windows, build-macos, sdist]
     steps:
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4.1.7
         with:
           name: wheels
       - name: Publish to PyPI

diff --git a/src/deduper.rs b/src/deduper.rs
@@ -297,13 +297,14 @@ fn write_attributes(
                         // skip empty documents if text_length is 0
                         for p in paragraphs {
                             let par_start = offset;
-                            offset += p.chars().count();
+                            let par_char_length = p.chars().count();
+                            offset += par_char_length;
                             if offset < text_length - 1 {
                                 offset += 1; // For the newline
                             }
                             let par_end = offset;
 
-                            if offset < min_content_length {
+                            if par_char_length < min_content_length {
                                 // skip length 0 paragraphs
                                 continue;
                             }