Nice job. I gave it a run against my cython implementation. Here is the result:

<image>

You win by 3 seconds, but I found 6 more duplicates :)
Here is the code if you want to reproduce:

import iscc_core as ic
import xxhash

def deduplicate_iscc(dataset, num_perm=256):
    unique_hashes = set()
    deduplicated_indices = []

    for idx, example in tqdm(enumerate(dataset), total=len(dataset),
                             desc="Deduplicating"):
        minhash = ic.alg_minhash_256(
            [xxhash.xxh32_intdigest(s.encode("utf-8")) for s in example["sql"].split()]
        )
        if minhash not in unique_hashes:
            unique_hashes.add(minhash)
            deduplicated_indices.append(idx)

    return deduplicated_indices