properly counting gzip crawl file list (#437)

nci · Jun 9, 2020 · e7b2b60 · e7b2b60
1 parent a96c393
commit e7b2b60
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/crawl/crawl_pipeline.sh b/crawl/crawl_pipeline.sh
@@ -47,7 +47,7 @@ fi
 
 crawl_file="$data_dir/${job_id}_gdal.tsv.gz"
 
-total_files=$(cat $file_list | wc -l)
+total_files=$(zcat $file_list | wc -l)
 batch_size=$(echo "batch=${total_files}/${conc_limit}; if(batch<1){batch=1;}; if(batch>10){batch=10;}; batch;"|bc)
 
 echo "INFO: file list to crawl: $file_list"