Split large files rather than compressing them

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 739a06ff1efb6f6aa1080b9ddc7c52ae34884604..d69c2a9118d86c2878d4ca5475d4d8e3a320b025 100644 (file)
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -15,8 +15,8 @@ update-lists:
     # fetch the last version of the blacklists
     - scripts/update-lists
 
-    # compress large files
-    - scripts/compress-large-files
+    # split large files
+    - scripts/split-large-files
 
     # update table in README file
     - bundle exec scripts/generate-readme < README.md.erb > README.md

diff --git a/scripts/compress-large-files b/scripts/compress-large-files
deleted file mode 100755 (executable)
index 8e6ed6f..0000000
--- a/scripts/compress-large-files
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/bin/bash
-
-set -euxo pipefail
-
-# compress files that are bigger than 1MB (GitHub limit is 100MB per file now)
-find blacklists -type f -size +1M -exec gzip -nf {} \;
-

diff --git a/scripts/split-large-files b/scripts/split-large-files
new file mode 100755 (executable)
index 0000000..2dafdfe
--- /dev/null
+++ b/scripts/split-large-files
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+set -euxo pipefail
+
+# split files that are larger than 50MB (GitHub limit is 100MB per file)
+find blacklists -type f -size +50M -exec bash -c 'split -b 50M -d -a 1 "{}" "{}." && rm "{}"' \;