fix typo

mlfoundations · May 24, 2024 · d077bb1 · d077bb1
1 parent 2da0d3b
commit d077bb1
Showing 1 changed file with 0 additions and 2 deletions.
diff --git a/open_lm/datapreprocess/ray/tokenize_shuffle.py b/open_lm/datapreprocess/ray/tokenize_shuffle.py
@@ -629,8 +629,6 @@ def main(args):
         )
     num_nodes = len(ray.nodes())
 
-    Sources = enum.Enum("Sources", {item["source"]: index for index, item in enumerate(data["sources"])})
-
     input_folders = args.input.split(",")
     input_paths = []
     for inp_folder in input_folders: