abdelmalek13
abdelmalek13
/ bilingual_tokenizer.py
Last active
February 3, 2025 12:57
# Bi-Lingual word Tokenization (Arabic and English) with Custom Regex
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import re | |
| import string | |
| import spacy | |
| import nltk | |
| from nltk.tokenize import (TweetTokenizer, word_tokenize) | |
| from camel_tools.tokenizers.word import simple_word_tokenize | |
| def bi_tokenizer(): |