BramVanroy · January 18, 2025 20:16 · BramVanroy · Jan 18, 2025
diff --git a/fw2-not-unique-ids.py b/fw2-not-unique-ids.py
 from typing import Counter
 from datasets import load_dataset
 from tqdm import tqdm


 ds = load_dataset("HuggingFaceFW/fineweb-2", "nld_Latn", split="train")
 ds_size = len(ds)

 print(f"Dataset size: {ds_size:,}")  #
 counts = Counter(ds["id"])

 counts = dict(sorted(counts.items(), key=lambda x: x[1], reverse=True))
 print(f"Number of unique IDs: {len(counts):,}")

 if len(counts) != ds_size:
    print("Number of unique IDs DOES NOT MATCH the dataset size.")
 else:
    print("Number of unique IDs matches the dataset size.")


 not_unique_counts = {k: v for k, v in tqdm(counts.items(), total=len(counts)) if v > 1}
 print(not_unique_counts)

 ds_not_unique = ds.filter(lambda x: x["id"] in not_unique_counts, num_proc=120)
 df = ds_not_unique.to_pandas()

 for uid, uiddf in df.groupby("id"): 
    text_is_identical = uiddf["text"].nunique() == 1
    date_is_identical = uiddf["date"].nunique() == 1
    file_path_is_identical = uiddf["file_path"].nunique() == 1
    id_is_identical = uiddf["id"].nunique() == 1

    id_str = f"{uid} - identical: text={text_is_identical}, date={date_is_identical}, file_path={file_path_is_identical}, id={id_is_identical}"
    print(f"{id_str}\n{'='*len(id_str)}")
    sample_idx = 0
    for _, row in uiddf.iterrows():
        sample_idx += 1
        print(f"Sample #{sample_idx} in {uid}:")
        print(f" - file_path: {row['file_path']}")
        print(f" - id: {row['id']}")
        print(f" - date: {row['date']}")
        # Only first 100 characters
        print(f" - text ([:100]): {row['text'][:100]}")
        print()
	from typing import Counter
	from datasets import load_dataset
	from tqdm import tqdm


	ds = load_dataset("HuggingFaceFW/fineweb-2", "nld_Latn", split="train")
	ds_size = len(ds)

	print(f"Dataset size: {ds_size:,}") #
	counts = Counter(ds["id"])

	counts = dict(sorted(counts.items(), key=lambda x: x[1], reverse=True))
	print(f"Number of unique IDs: {len(counts):,}")

	if len(counts) != ds_size:
	print("Number of unique IDs DOES NOT MATCH the dataset size.")
	else:
	print("Number of unique IDs matches the dataset size.")


	not_unique_counts = {k: v for k, v in tqdm(counts.items(), total=len(counts)) if v > 1}
	print(not_unique_counts)

	ds_not_unique = ds.filter(lambda x: x["id"] in not_unique_counts, num_proc=120)
	df = ds_not_unique.to_pandas()

	for uid, uiddf in df.groupby("id"):
	text_is_identical = uiddf["text"].nunique() == 1
	date_is_identical = uiddf["date"].nunique() == 1
	file_path_is_identical = uiddf["file_path"].nunique() == 1
	id_is_identical = uiddf["id"].nunique() == 1

	id_str = f"{uid} - identical: text={text_is_identical}, date={date_is_identical}, file_path={file_path_is_identical}, id={id_is_identical}"
	print(f"{id_str}\n{'='*len(id_str)}")
	sample_idx = 0
	for _, row in uiddf.iterrows():
	sample_idx += 1
	print(f"Sample #{sample_idx} in {uid}:")
	print(f" - file_path: {row['file_path']}")
	print(f" - id: {row['id']}")
	print(f" - date: {row['date']}")
	# Only first 100 characters
	print(f" - text ([:100]): {row['text'][:100]}")
	print()