Add User filter on wiki page (#13)

* add filter

* rename filter

* remove unwanted import

clean.py

@@ -3,7 +3,7 @@
 from datasets import Dataset, load_dataset, load_from_disk
 
 from datasets.utils.logging import set_verbosity_info
-from clean_helpers import filter_wiki_non_text_type
+from clean_helpers import filter_wiki_user_titles, filter_wiki_non_text_type
 
 set_verbosity_info()
 logger = logging.getLogger(__name__)
@@ -13,7 +13,8 @@
 MAPS = {}
 # Filter functions
 FILTERS = {
-    "filter_wiki_non_text_type": filter_wiki_non_text_type
+    "filter_wiki_user_titles": filter_wiki_user_titles,
+    "filter_wiki_non_text_type": filter_wiki_non_text_type,
 }
 
 assert set(MAPS.keys()).isdisjoint(set(FILTERS.keys()))

clean_helpers/__init__.py

@@ -1 +1 @@
-from .filter_wiki_meta import filter_wiki_non_text_type
+from .filter_wiki_meta import filter_wiki_user_titles, filter_wiki_non_text_type

clean_helpers/filter_wiki_meta.py

@@ -1,2 +1,6 @@
+def filter_wiki_user_titles(examples):
+    return [not eval(meta)["title"].startswith("User ") for meta in examples["meta"]]
+
 def filter_wiki_non_text_type(examples):
     return [eval(meta)["type"] == "text" for meta in examples["meta"]]
+

slurm/cleaning_filtering/00_test_filter_user_titles.sh

@@ -0,0 +1,12 @@
+conda activate datacatalog
+
+CATALOGUE_DATA_REPO="/home/lucile/code/catalogue_data"
+
+cd $CATALOGUE_DATA_REPO
+
+python clean.py \
+    --dataset-path bigscience-catalogue-lm-data/lm_en_wikinews_filtered \
+    --maps-and-filters filter_wiki_user_titles \
+    --save-path /home/lucile/data/result_filtering_cleaning/lm_en_wikinews_filtered.jsonl \
+    --num-proc 4 \
+    --batch-size 100