inherited from names_database

.gitignore

@@ -1,3 +1 @@
-__pycache__\*
-__pycache__/spacy_analyzer.cpython-310.pyc
-__pycache__/spacy_recognizer.cpython-310.pyc
+__pycache__/*

Pipfile

@@ -14,6 +14,7 @@ streamlit = "==1.17.0"
 tokenizers = "==0.12.1"
 torch = "==1.12.0"
 en-student-name-detector = {file = "https://huggingface.co/langdonholmes/en_student_name_detector/resolve/main/en_student_name_detector-any-py3-none-any.whl"}
+names-dataset = "*"
 
 [dev-packages]
 

Pipfile.lock

@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "95f6932d0b58210f580f1426df3f909ce879d49cb04320ceff02998ac500d614"
+            "sha256": "6a4aa8c782c5b5fd8f5f0b3d7ba6cb6541f37295823bdee26d3fd575533c5999"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -463,39 +463,46 @@
             "markers": "python_version >= '3.6'",
             "version": "==1.0.9"
         },
+        "names-dataset": {
+            "hashes": [
+                "sha256:69eea12c9d97e1ae32b6db955bb9b39f7816eb2727d3c6abc726cb475ad160ac"
+            ],
+            "index": "pypi",
+            "version": "==3.1.0"
+        },
         "numpy": {
             "hashes": [
-                "sha256:0044f7d944ee882400890f9ae955220d29b33d809a038923d88e4e01d652acd9",
-                "sha256:0e3463e6ac25313462e04aea3fb8a0a30fb906d5d300f58b3bc2c23da6a15398",
-                "sha256:179a7ef0889ab769cc03573b6217f54c8bd8e16cef80aad369e1e8185f994cd7",
-                "sha256:2386da9a471cc00a1f47845e27d916d5ec5346ae9696e01a8a34760858fe9dd2",
-                "sha256:26089487086f2648944f17adaa1a97ca6aee57f513ba5f1c0b7ebdabbe2b9954",
-                "sha256:28bc9750ae1f75264ee0f10561709b1462d450a4808cd97c013046073ae64ab6",
-                "sha256:28e418681372520c992805bb723e29d69d6b7aa411065f48216d8329d02ba032",
-                "sha256:442feb5e5bada8408e8fcd43f3360b78683ff12a4444670a7d9e9824c1817d36",
-                "sha256:6ec0c021cd9fe732e5bab6401adea5a409214ca5592cd92a114f7067febcba0c",
-                "sha256:7094891dcf79ccc6bc2a1f30428fa5edb1e6fb955411ffff3401fb4ea93780a8",
-                "sha256:84e789a085aabef2f36c0515f45e459f02f570c4b4c4c108ac1179c34d475ed7",
-                "sha256:87a118968fba001b248aac90e502c0b13606721b1343cdaddbc6e552e8dfb56f",
-                "sha256:8e669fbdcdd1e945691079c2cae335f3e3a56554e06bbd45d7609a6cf568c700",
-                "sha256:ad2925567f43643f51255220424c23d204024ed428afc5aad0f86f3ffc080086",
-                "sha256:b0677a52f5d896e84414761531947c7a330d1adc07c3a4372262f25d84af7bf7",
-                "sha256:b07b40f5fb4fa034120a5796288f24c1fe0e0580bbfff99897ba6267af42def2",
-                "sha256:b09804ff570b907da323b3d762e74432fb07955701b17b08ff1b5ebaa8cfe6a9",
-                "sha256:b162ac10ca38850510caf8ea33f89edcb7b0bb0dfa5592d59909419986b72407",
-                "sha256:b31da69ed0c18be8b77bfce48d234e55d040793cebb25398e2a7d84199fbc7e2",
-                "sha256:caf65a396c0d1f9809596be2e444e3bd4190d86d5c1ce21f5fc4be60a3bc5b36",
-                "sha256:cfa1161c6ac8f92dea03d625c2d0c05e084668f4a06568b77a25a89111621566",
-                "sha256:dae46bed2cb79a58d6496ff6d8da1e3b95ba09afeca2e277628171ca99b99db1",
-                "sha256:ddc7ab52b322eb1e40521eb422c4e0a20716c271a306860979d450decbb51b8e",
-                "sha256:de92efa737875329b052982e37bd4371d52cabf469f83e7b8be9bb7752d67e51",
-                "sha256:e274f0f6c7efd0d577744f52032fdd24344f11c5ae668fe8d01aac0422611df1",
-                "sha256:ed5fb71d79e771ec930566fae9c02626b939e37271ec285e9efaf1b5d4370e7d",
-                "sha256:ef85cf1f693c88c1fd229ccd1055570cb41cdf4875873b7728b6301f12cd05bf",
-                "sha256:f1b739841821968798947d3afcefd386fa56da0caf97722a5de53e07c4ccedc7"
+                "sha256:003a9f530e880cb2cd177cba1af7220b9aa42def9c4afc2a2fc3ee6be7eb2b22",
+                "sha256:150947adbdfeceec4e5926d956a06865c1c690f2fd902efede4ca6fe2e657c3f",
+                "sha256:2620e8592136e073bd12ee4536149380695fbe9ebeae845b81237f986479ffc9",
+                "sha256:2eabd64ddb96a1239791da78fa5f4e1693ae2dadc82a76bc76a14cbb2b966e96",
+                "sha256:4173bde9fa2a005c2c6e2ea8ac1618e2ed2c1c6ec8a7657237854d42094123a0",
+                "sha256:4199e7cfc307a778f72d293372736223e39ec9ac096ff0a2e64853b866a8e18a",
+                "sha256:4cecaed30dc14123020f77b03601559fff3e6cd0c048f8b5289f4eeabb0eb281",
+                "sha256:557d42778a6869c2162deb40ad82612645e21d79e11c1dc62c6e82a2220ffb04",
+                "sha256:63e45511ee4d9d976637d11e6c9864eae50e12dc9598f531c035265991910468",
+                "sha256:6524630f71631be2dabe0c541e7675db82651eb998496bbe16bc4f77f0772253",
+                "sha256:76807b4063f0002c8532cfeac47a3068a69561e9c8715efdad3c642eb27c0756",
+                "sha256:7de8fdde0003f4294655aa5d5f0a89c26b9f22c0a58790c38fae1ed392d44a5a",
+                "sha256:889b2cc88b837d86eda1b17008ebeb679d82875022200c6e8e4ce6cf549b7acb",
+                "sha256:92011118955724465fb6853def593cf397b4a1367495e0b59a7e69d40c4eb71d",
+                "sha256:97cf27e51fa078078c649a51d7ade3c92d9e709ba2bfb97493007103c741f1d0",
+                "sha256:9a23f8440561a633204a67fb44617ce2a299beecf3295f0d13c495518908e910",
+                "sha256:a51725a815a6188c662fb66fb32077709a9ca38053f0274640293a14fdd22978",
+                "sha256:a77d3e1163a7770164404607b7ba3967fb49b24782a6ef85d9b5f54126cc39e5",
+                "sha256:adbdce121896fd3a17a77ab0b0b5eedf05a9834a18699db6829a64e1dfccca7f",
+                "sha256:c29e6bd0ec49a44d7690ecb623a8eac5ab8a923bce0bea6293953992edf3a76a",
+                "sha256:c72a6b2f4af1adfe193f7beb91ddf708ff867a3f977ef2ec53c0ffb8283ab9f5",
+                "sha256:d0a2db9d20117bf523dde15858398e7c0858aadca7c0f088ac0d6edd360e9ad2",
+                "sha256:e3ab5d32784e843fc0dd3ab6dcafc67ef806e6b6828dc6af2f689be0eb4d781d",
+                "sha256:e428c4fbfa085f947b536706a2fc349245d7baa8334f0c5723c56a10595f9b95",
+                "sha256:e8d2859428712785e8a8b7d2b3ef0a1d1565892367b32f915c4a4df44d0e64f5",
+                "sha256:eef70b4fc1e872ebddc38cddacc87c19a3709c0e3e5d20bf3954c147b1dd941d",
+                "sha256:f64bb98ac59b3ea3bf74b02f13836eb2e24e48e0ab0145bbda646295769bd780",
+                "sha256:f9006288bcf4895917d02583cf3411f98631275bc67cce355a7f39f8c14338fa"
             ],
             "markers": "python_version >= '3.10'",
-            "version": "==1.24.1"
+            "version": "==1.24.2"
         },
         "packaging": {
             "hashes": [
@@ -542,10 +549,10 @@
         },
         "phonenumbers": {
             "hashes": [
-                "sha256:2e3fd1f3fde226b289489275517c76edf223eafd9f43a2c2c36498a44b73d4b0",
-                "sha256:6eb2faf29c19f946baf10f1c977a1f856cab90819fe7735b8e141d5407420c4a"
+                "sha256:1531b42c8c49a1f06b08598441bf1f11fe2618f707c6fc96b581b44aa4f2b0e3",
+                "sha256:f8bd92975ba7463b7828ae2f95e1037b7e0ab8f023e9e8ffb7c560fd7f5d66d7"
             ],
-            "version": "==8.13.5"
+            "version": "==8.13.6"
         },
         "pillow": {
             "hashes": [
@@ -737,6 +744,13 @@
             "markers": "python_version >= '3.7'",
             "version": "==11.0.0"
         },
+        "pycountry": {
+            "hashes": [
+                "sha256:b2163a246c585894d808f18783e19137cb70a0c18fb36748dc01fc6f109c1646"
+            ],
+            "markers": "python_version >= '3.6' and python_version < '4'",
+            "version": "==22.3.5"
+        },
         "pycryptodome": {
             "hashes": [
                 "sha256:04779cc588ad8f13c80a060b0b1c9d1c203d051d8a43879117fe6b8aaf1cd3fa",
@@ -1070,11 +1084,11 @@
         },
         "setuptools": {
             "hashes": [
-                "sha256:a7687c12b444eaac951ea87a9627c4f904ac757e7abdc5aac32833234af90378",
-                "sha256:e261cdf010c11a41cb5cb5f1bf3338a7433832029f559a6a7614bd42a967c300"
+                "sha256:16ccf598aab3b506593c17378473978908a2734d7336755a8769b480906bec1c",
+                "sha256:b440ee5f7e607bb8c9de15259dba2583dd41a38879a7abc1d43a71c59524da48"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==67.1.0"
+            "version": "==67.2.0"
         },
         "six": {
             "hashes": [
@@ -1480,11 +1494,11 @@
         },
         "zipp": {
             "hashes": [
-                "sha256:73efd63936398aac78fd92b6f4865190119d6c91b531532e798977ea8dd402eb",
-                "sha256:9eb0a4c5feab9b08871db0d672745b53450d7f26992fd1e4653aa43345e97b86"
+                "sha256:23f70e964bc11a34cef175bc90ba2914e1e4545ea1e3e2f67c079671883f9cb6",
+                "sha256:e8b2a36ea17df80ffe9e2c4fda3f693c3dad6df1697d3cd3af232db680950b0b"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==3.12.0"
+            "version": "==3.13.0"
         }
     },
     "develop": {}

data/ascii_fb_names_small.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:baf5cf2fa43dc172c613f72793641f668e33c30b4e23932616de36cc0ce3447d
+size 33601747

match_replace.py

@@ -1,69 +1,31 @@
 import pandas as pd
 
-def replace_name_old(country_code, gender, f_l, original_name, fb_df):
-    """
-    Receiving country, gender, first_last name, and the original name.
-    Match with a name that matches gender and country, and is randomly retrieved from the
-    facebook dataset.
-    Compare the surrogate name with the original name to make sure they are different.
-    Return the surrogate name in a form of string.
-    f_l: F or L for first or last name -> str
-    """
-    # prioritizing GENDER over country?
-    # it is a very large dataset so can take long, how to improve the speed?
-    # Q: If want to get a whole name at a time? (just combining)
-    # Q: If only get initials? (change to other letters which should be easy)
-    # translating gender code
-    ###### randomly find a match in the data set! And a return a similar one
-    # if gender == 'male':
-    #     gender = 'M'
-    # elif gender == 'female':
-    #     gender = 'F'
-    # else:
-    #     gender = None
-
-    surrogate_name = original_name
-    # checking whether the surrogate name and the original name is the same
-    # using the while loop
-    # TODO: [Old version] the order of gender and country need to be changed
-    while(surrogate_name == original_name):
-        # situation when gender can be matched
-        if not gender:
-            gender_df = fb_df[fb_df["gender"] == gender]
-            gender_c_df = gender_df[gender_df["country"] == country_code]
-            # situations: whether country code can be matched
-            if gender_c_df.shape[0] > 0:
-                surrogate_name = gender_c_df[f_l].sample(n=1).to_string()
-            # if gender match, country not match: randomly return from gender df
-            else:
-                surrogate_name = gender_df[f_l].sample(n=1).to_string()
-        else:
-            # situation when gender cannot be match: gender is None
-            country_df = fb_df[fb_df["country"] == country_code]
-            # situation when country can be matched
-            if country_df.shape[0] > 0:
-                surrogate_name = country_df[f_l].sample(n=1).to_string()
-            # situation when neither gender nor country can be matched
-            # randomly return one name from the whole dataset
-            else:
-                surrogate_name = fb_df[f_l].sample(n=1).to_string()
-
-    return surrogate_name
-
-def match_entity(original_info, entity):
-    # TODO: need refinement for each kind of entity
-    if entity == 'STUDENT':
-    # TODO: here, change between 1 and 2
-        return match_name_2(original_info)
-    elif entity == 'EMAIL_ADDRESS':
-        return '[email protected]'
-    elif entity == 'PHONE_NUMBER':
-        #TODO: specific form of number will be returned for consistency
-        return '000-000-0000'
-    elif entity == 'URL':
-        return 'google.com'
-    else:
-        pass
+from names_database import NameDatabase
+
+names_db = NameDatabase
+
+def describe_name(first_names, last_names):
+    gender = names_db.get_gender() if first_names else None
+    country = names_db.get_country() if last_names else None
+    return gender, country
+
+def split_name(all_names):
+    '''Splits name into parts.
+    If one token, assume it is a first name.
+    If two tokens, first and last name.
+    If three tokens, one first name and two last names.
+    If four tokens, two first names and two last names.'''
+    match all_names.split():
+        case [first]:
+            return first, None
+        case [first, last]:
+            return first, last
+        case [first, last_1, last_2]:
+            return first, ' '.join((last_1, last_2))
+        case [first_1, first_2, last_1, last_2]:
+            return ' '.join((first_1, first_2)), ' '.join((last_1, last_2))
+        case _:
+            return None, None
 
 def match_name(original_name):
     # FIXME: take too LONG time to run (large df used multi-times), how to improve
@@ -74,7 +36,6 @@ def match_name(original_name):
     # FIXME: since it is completely random, the same original name may be diff after replacing. How to know whether the two names is the same person?
     first_name = original_name.split()[0]
     global fb_df
-    fb_df = pd.read_parquet('ascii_fb_names_small.parquet')
     names = fb_df[fb_df['first']==first_name]
     if not names.empty:
         name_df = names.sample(n=1)

names_database.py

@@ -0,0 +1,28 @@
+from names_dataset import NameDataset, NameWrapper
+from typing import Optional
+
+class NameDatabase(NameDataset):
+    def __init__(self) -> None:
+        super().__init__()
+        self.names = pd.read_parquet('ascii_fb_names_small.parquet')
+
+    def get_random_name(
+            self,
+            country: Optional[str] = None,
+            gender: Optional[str] = None
+    ):
+        '''country: ISO country code in 'alpha 2' format
+        gender: "M" or "F"
+        '''
+        names_view = self.names
+        if country:
+            names_view = names_view[names_view['country'] == country]
+        if gender:
+            names_view = names_view[names_view['gender'] == gender]
+        return names_view.sample(weights=names_view.count)
+    
+    def get_gender(first_names: str):
+        return NameWrapper(self.search(first_names)).gender
+    
+    def get_country(last_names: str):
+        return NameWrapper(self.search(last_names)).country