Upload 34 files

.gitattributes

@@ -38,3 +38,6 @@ benchmarks/DeepLoc/membrane_type_train.csv filter=lfs diff=lfs merge=lfs -text
 benchmarks/DeepLoc/OG_membrane_type_all.csv filter=lfs diff=lfs merge=lfs -text
 data/uniref/100k_seqs/train.csv filter=lfs diff=lfs merge=lfs -text
 data/uniref/200k_seqs/train.csv filter=lfs diff=lfs merge=lfs -text
+benchmarks/Supervised/Localization/true_deeploc2.0_cell-local_train-val.csv filter=lfs diff=lfs merge=lfs -text
+benchmarks/Supervised/Membrane[[:space:]]Type/membrane_type_train.csv filter=lfs diff=lfs merge=lfs -text
+benchmarks/Supervised/Membrane[[:space:]]Type/unsplit_membrane_type_all.csv filter=lfs diff=lfs merge=lfs -text

benchmarks/Generation/ProtGPT2/protgpt2_finetune.py

@@ -0,0 +1,70 @@
+import pandas as pd
+import os
+import subprocess
+from transformers import AutoTokenizer, AutoModelForCausalLM
+
+
+# Format sequence inputs based on ProtGPT fine-tuning requirements
+def modify_sequences(sequence):
+  modified_sequence = sequence.upper()
+  modified_sequence = '\n'.join([modified_sequence[i:i+60] for i in range(0, len(modified_sequence), 60)])
+
+  fasta = "<|endoftext|>"
+  modified_sequence = fasta + "\n" + modified_sequence
+
+  return modified_sequence
+
+# Function to save sequences to txt files
+def to_txt_file(df, filename):
+  with open(filename, 'w') as f:
+    for sequence in df['Sequence']:
+      f.write(sequence + '\n')
+
+
+# Modify the sequences
+path = "/workspace/sg666/MDpLM"
+
+train = pd.read_csv(path + "/data/membrane/train.csv")
+val = pd.read_csv(path + "/data/membrane/val.csv")
+test = pd.read_csv(path + "/data/membrane/test.csv")
+
+train = pd.concat([train, val])
+
+train['Sequence'] = train['Sequence'].apply(modify_sequences)
+test['Sequence'] = test['Sequence'].apply(modify_sequences)
+
+
+# Save the modified sequences as txt files
+to_txt_file(train, path + '/benchmarks/Generation/ProtGPT2/protgpt2_train.txt')
+to_txt_file(test, path + '/benchmarks/Generation/ProtGPT2/protgpt2_test.txt')
+
+
+tokenizer = AutoTokenizer.from_pretrained("nferruz/ProtGPT2")
+model = AutoModelForCausalLM.from_pretrained("nferruz/ProtGPT2")
+
+finetune_protgpt2_command = [
+    "python", "run_clm.py",
+    "--model_name_or_path", "nferruz/ProtGPT2",
+    "--train_file", "protgpt2_train.txt",
+    "--validation_file", "protgpt2_test.txt",
+    "--tokenizer_name", "nferruz/ProtGPT2",
+    "--num_train_epochs", "10",
+    "--logging_steps", "1",
+    "--logging_dir", "test",
+    "--do_train",
+    "--do_eval",
+    "--output_dir", "/workspace/sg666/MDpLM/benchmarks/Generation/ProtGPT2/finetuned_models",
+    "--overwrite_output_dir",
+    "--learning_rate", "3e-04",
+    "--per_device_train_batch_size", "2",
+    "--evaluation_strategy", "epoch"
+]
+
+try:
+    result = subprocess.run(finetune_protgpt2_command, check=True, text=True, capture_output=True)
+except subprocess.CalledProcessError as e:
+    print("Command failed with the following error:")
+    print(e.stderr)  # Print standard error output
+    print("Command output:")
+    print(e.stdout)  # Print standard output if needed
+

benchmarks/Generation/ProtGPT2/protgpt2_generate.py

@@ -0,0 +1,55 @@
+from transformers import pipeline
+from transformers import AutoModelForCausalLM, AutoTokenizer
+import math
+import torch
+import sys
+import pandas as pd
+
+# Function to calculate perplexity of each generated sequence
+def calculate_perplexity(sequence, model, tokenizer):
+    sequence = "<|endoftext|>"  + sequence + "<|endoftext|>"
+    input_ids = torch.tensor(tokenizer.encode(sequence)).unsqueeze(0)
+    input_ids = input_ids.to(device)
+    with torch.no_grad():
+        outputs = model(input_ids, labels=input_ids)
+    loss, _ = outputs[:2]
+    return math.exp(loss)
+
+if __name__ == "__main__":
+    device = torch.device("cuda" if torch.cuda.is_available() else 'cpu')
+    path = "/workspace/sg666/MDpLM/benchmarks/Generation/ProtGPT2"
+
+    # Load fine-tuned model and tokenizer
+    model_path = path + "/finetuned_models/checkpoint-4510"
+    model = AutoModelForCausalLM.from_pretrained(model_path)
+    tokenizer = AutoTokenizer.from_pretrained(model_path)
+
+    # Generate sequences
+    protgpt2 = pipeline('text-generation', model=model_path, device=device)
+    sequences = protgpt2("", max_length=100, do_sample=True, top_k=950, repetition_penalty=1.5, num_return_sequences=100, eos_token_id=0)
+
+    # Store generated sequences and their associated perplexities
+    generated_sequences = []
+    perplexities = []
+
+
+    # Calculate PPL for sequences
+    for item in sequences:
+        raw_sequence = item['generated_text']
+        ppl = calculate_perplexity(raw_sequence, model.to(device), tokenizer)
+        generated_sequences.append(raw_sequence)
+        perplexities.append(ppl)
+
+    # Clean the generated sequences
+    cleaned_sequences = [seq.replace('\n', '').replace('<|endoftext|>', '') for seq in generated_sequences]
+
+    # Create df with cleaned sequences and perplexities
+    df = pd.DataFrame({"Sequence": cleaned_sequences, "Perplexity": perplexities})
+    df.sort_values(by='Perplexity', inplace=True)
+
+    # Save results
+    df.to_csv(path + "/protgpt2_generated_sequences.csv", index=False)
+
+    # View the average de novo generation perplexity
+    avg_generation_ppl = df.loc[:, 'Perplexity'].mean()
+    print(f'Average de novo generation perplexity: {avg_generation_ppl}')

benchmarks/Generation/ProtGPT2/protgpt2_generated_sequences.csv

@@ -0,0 +1,101 @@
+Sequence,Perplexity
+LAPSVVTGVAQSSPLTIVTNPKEPRQPVPASDGADYLKTIPGFAVIRNGGSNGDPVLRGMFGSRLNILTNGGMMLGACPNRMDAPTSYISPETYDKLTVIKGPQTVLWGPGASAGTILFEREPERFGELGSRVNASLLAGSNGRFDKVLDAAAGNRLGYLRFTGNHAQSDDYEDGAGNTVPSRWKKWNGDVAVGWTPDEDTLIELTAGKGDGEARYAGRGMDGSQFKRESLGLRFVKSNVSDVLEKVEAQVYYNYADHIMDNFRLRTPDPSMPMT,2.6532732777535712
+MPNFFIDRPIFAWVIAIIIMLAGGLAILKLPVAQYPTIAPPAVTISASYPGADAKTVQDTTVQIIEQNLNGLDNLLYMSSTSDDSGNATITITFAPGTNPDIAQVQVQNKLSLATPILPQAVQRQGVSVEKSSSSFLMVVGVINTDGTMTQEDISDYVAANMKDAISRTSGVGDVQLFGSQYAMRIWMNPNELNKFQLTPVDVITAIKAQNAQVAAGQLGGTPPVKGQQLNASIIAQTRLTSTEEFGKILLKVNQDGSRVLLRDVAKIELGGENYDIIAEFNGQPASGLGIKLATG,2.829348107084168
+MAYRSTTLLALLALVLLYLVSGALVFRALEQPHEQQAQRELGEVREKFLRAHPCVSDQELGLLIKEVADALGGGADPETQSTSAWDLGSAFFFSGTIITTIGYGNVALRTDAGRLFCIFYAAXFGIPFTLLFLTAVGDRLGSSLRHGIGHIEAIFLKWHVPPELVRVLSEMLFLLVGCLLFVLTPTFVFCYMEDWSKLEAIYFVIVTLTTVGFGDYVAGADPRQDSPAYQPLVWFWILLGLAYFASVSAML,3.119025307842878
+MPNFFIDRPIFAWVIAIIIMLAGGLAILKLPVAQYPTIAPPAVTISASYPGADAKTVQDTTVQIIEQQMNGLDGLRYISSNSAGNGQASIQLNFEQGVDPDIAQVQVQNKLQLAMPLLPQAVKEQGVSVEKSSSSFLMVVGVINTDGTMTQEDISDYVAANMKDAISRTSGVGDVQLFGSQYAMRIWMNPNELNKFQLTPVDVITAIKAQNAQVAAGQLGGTPPVKGQQLNASIIAQTRLTSTEEFGKILLKVNQDGSRVLLRDVAKIELGGENYDIIAEFNGQPASGLG,3.775355043694786
+LFLTMAEAQLRYKTTEECLAYFGVSETTGLTPDQVKRHLEKYGHNELPAEEGKSLWELVIEQFEDLLVRILLLAACISFVLAWFEEGEETVTAFVEPFIILLILIANAIVGVWQERNAENAIEALKEYEPEMGKVYRADRKSVQRIKARDIVPGDIVEVAVGDKVPADIRILSIKSTTLRVDQSILTGESVSVAKSSDAVPDPRAVNQDKKNMLFSGTNIAAGKALGIVATTGVSTEIGKIRDQMAATEQDKTPLQQKLDEFGEQLSKVISLICIAVWLINIGHFNDPVHGGSWI,4.136701078251139
+MPNFFIDRPIFAWVIAIIIMLAGGLAILKLPVAQYPTIAPPAVTISASYPGADAKTVQDTTVQIIEQNMNGIDNLMYMSSNSDSTGTAQITLTFESGTDADIAQVQVQNKLQLAMPLLPQAVQQQGVSVEKSSSSFLMVVGVINTDGTMTQEDISDYVAANMKDAISRTSGVGDVQLFGSQYAMRIWMNPNELNKFQLTPVDVITAIKAQNAQVAAGQLGGTPPVKGQQLNASIIAQTRLTSTEEFGKILLKVNQDGSRVLLRDVAKIELGGENYDIIAEFNGQPASGLG,4.210716900525416
+MPNFFIDRPIFAWVIAIIIMLAGGLAILKLPVAQYPTIAPPAVTISASYPGADAKTVQDTTVQIIEQQMNGLDGLRYISSNSAGNGQASIQLTFESGTDADIAQVQVQNKLQLAMPLLPQEVAQQGVSVEKSSSSFLMVVGVINTDGTMTQEDISDYVAANMKDAISRTSGVGDVQLFGSQYAMRIWMNPNELNKFQLTPVDVITAIKAQNAQVAAGQLGGTPPVKGQQLNASIIAQTRLTSTEEFGKILLKVNQDGSRVLLRDVAKIELGGENYDIIAEFNGQPAS,4.526996795741569
+MLKIIIPTTMLLPMTWMSKHNMIWINATVHSLLISLISLSLLNQLGENSLNFSLTFFSDSLSAPLLVLTTWLLPLMLMASQSHLSKETTTRKKLYITMLLILLQLFLIMTFTATELILFYIFESASLPTLLMITRWGNQTERLNAGLYFLMYTLAGSLPLLVALVYIQNTTGSLNFLIIHWSTHTSASFVSQTLLLMAWMAAMAVMAKMPLYGVHLWLPKAHVEAPIAGSMVLAAVLLKLGGYGMLRITTILNPLTNYMAYPFLMLCLWGMI,4.629232424547782
+AKFINRWLFSTNHKDIGTLYLLFGAWAGMVGTALSLLIRAELGQPGTLLGDDQIYNVVVTGHAFVMIFFMVMPIMIGGFGNWLVPLMIGAPDMAFPRMNNMSFWLLPPSFLLLLASSMVEAGAGCGWTVYPPLAGNLAHAGASVDLTIFSLHLAGVSSILGAINFITTIINMKPPAMSQYQTPLFVWSVMITAVLLLLSLPVLAAGITMLLTDRNLNTTFFDPAGGGDPILYQHLFWFFGHPXVLILILPFFGIVTEASAIPRIFNWMVTFHGQLMYHHMWIIGVL,5.0608380016313275
+LVEKDPIKTSFEKWAKPGHFSRTLAKGPNTTTWIWNLHADAHDFDSYTSDLEEISRKVFSAHFGHLAVVTIWLSGMIFHGAKFSNYEAWLSDPLNVRPSAQVVWPLVGQDILNGDMGDGTYNGFQVMTSGLFQLWRASGITNEYQLYCTAIGGLVMAALMLFAGWFHYHKAAPKLAWFQDVETALNHHLSGLLGLGCLSWAGHQIHVSLPVNKLLDAGVAAKDIPLPHEFILDPAKFASLLPGLTQGLTPFFTLNWSEYSDFLTFKGGLNPVTGGL,5.597917119515088
+MVRKVYVTLQGKVQGVFFRAHTQATAKQLGVVGWVRNTSDGTVEGEAQGPADKVDEMINWLHRGPPQAQIESHEFNSEKKELEAFSSFHIRY,5.635017933300935
+EFGFWEIKFPEYLKGRPTTGRPEWVQDVDLVNKWAVPGLNPPHHFSPPVNLTGVEDTLPVSWVMVSMVVGFVLIVATAGNILVIIAVFTSRALKAPQNLFLVSLASADILVATLVIPFAMANEVMGYWYFGKAWCEIYLALDVLFCTSSAWHLCAISLDRYWSITQAIEYNLKRTPRRTKAIIITVWVISAVISFPPRCEINDQKWYYVISSCIGSFFAPCLIMILVYVRIYQIAKRRTRDLSRKSGRPSLLSEVHAAKSLAIL,6.370992471309986
+MVYVSRISVFAFLGALASVAYGQVTPPNFGTEQDRVNFTKQIVPVLKEKCVVCHGPDKTKGKLRLDLRIEAFKGGESGESIDVIPGDPENSELLERITSKDPEFRMPPKSEHKPLTEAEIALLKQWILEGAKYDPAWAFTPPKRTDLPKVKRDEWAKNDVDRFILAKLESEGLTPNPEADKATLIRRVTLDLTGLPPTPAEVDAFLADKSPNAYEKVVDRLLASPHFGERWGRHWLDVARWAESNGFERNTIRNIWSYRDWVIKALNDDVPYDQFTVEQL,7.0499259667086145
+SSNAKTVLITGGTGFVGRALVKRLLSTTKHTIVVPYREEADLHDVKVLQVKGDLRDAASLDAAFEGVDCVFHLASYGMSGPEMFELNVEGTRNVVEACLRHGVRRLIHVSSIAVMGEPSDHPRREADESLPARQATAYAKSKVEAERIVLEANGSDGLETVVVRPPMVWGPGDTQFLPRLVRMARRGLRPVIGNGKSLVSMVYIDNLVDGLIAAMDHPEARGKTYFLSNDGHASQREFIETVARAIGRPAPKLTLPVPVLYWAARLLG,7.429969652690046
+SPELIEQLLQNYLQLPDAEKRKVADQLQTSNIRYCYLLASEKGWLDRVESCLAAEGCDVLQPDHTGRNLLQVVASVSPDHTARLIRALLARGADVHAQDSLGNTVLHILILQPNKTFACQMYNEILILGAKLCPTVNLEAVLNHQGLTPFKLAGVEGNTVMFQHLMQKRKHVQWTCGPLTSTLYDLTEIDSSGDDQSLLELIVTTKKREARQILEQTPVKELVSLKWKRYGRPYFCVLGAIYILYIICFTMCCVYRPLKPRITNRTNPRDNMTSLEL,7.910941817905356
+ADVNLNARDLHGMTPLHLAAKNGHDKVVQLLLKKGALVNIQDKLGSTPLLEAIRGRREDTVKLLVEHGADIRAQDSLGNTVLHILILQPENSTSLKFAEMLYDMILLRSGTWELETTQPNDGLTALQLAAKMGKAEILKYILSREIKEKPLRSLSRKFTDWAYGPVSSSLYDLTNVDSSGNTVLHAMIMVADNTPQNSRFVKQMYNLLLSKGARLCPNVPNHQGLTPFKLAGVEGNIVMQEILRGTTISIPFTCITCGKKDTRFRGMSCEN,8.179497248919981
+DPFNNFFRRSKIAVCGLVFFVLFIIYMVLGSMIFSAIERDHEQQAQRELGEVREKFLISHPCVSDQELGVLIEEVADALGGGADPETQSTSAWDLGSAFFFSGTIITTIGYGNVALRTDTMGRLFCIFYALVGIPLFGILLAGVGDRLGSSLRHGIGHIEAIFLKWHVPPGLVRVLSAMLFLAIGCLLFVTLPAYVFSHMEDWSKLEAIYFVIVTLTTVGFGDYVAGADPRQDSPQYQPLVWFWILL,8.306921086116862
+GPQSFVHFTKQSLALIEQRIAERKSKEPKPSSDLEAGKQLPFIYGDIPPGMVSEPLEDLDPYYADKKTFIVLNKGKTIFRFNATPALYMLSPFSPLRRISIKILVHSLFSMLIMCTILTNCIFMTMNNPPDWTKNVEYTFTGIYTFESLVKILARGFCVGEFTFLRDPWNWLDFVVIVFAYVTEFVVAEFVSFSALRAFRVLRALKTISVIPGLKTIVGALIQSVKKLSDVMILTVFCLSVFALIGLQLFMGNLRKKCFFPDG,8.471762198050271
+MLKIIIPTTMLLPMTWMSKHNMIWINATVHSLLISLISLSLLNQLGENSLNFSLTFFSDSLSAPLLVLTTWLLPLMLMASQSHLSKETTTRKKLYITQLILLQLFLIMTFTATELILFYIFESATLLPTLLIILRWGYQPERLQAGLYFLFYTLIGGVLVLLSILMIYVNTNSLLIHTLPMFNSTMETSLYTKIMWFACMMAFPTKMGLFPIHMWLPVVHSESPLAGSCILAGILLKLGGYGMMRVVTILNPLTNYMAYPFLML,8.583127806228307
+MVLRLVVLALLCWTPGLWAQQADTLTLDEVVVTATRSEQNLQDVPASVSVITAEDLQRQAPRTLGEALRYVPGVFLDGTGRTNGQDINMRGYDHRGVLVLVDGIRQGTDTGHLNGTFLDPALIKRVEIVRGPSAALYGNGAAGGVVNFITRQPSDQLTGSVRLNTSLPQHDGDNSQQFYSLMAGNRLGEEGKLGMLASFSRQEKGQARDGAGNDIASLDEDSLSGKLLWQLTPEQQLDFSLDHYRFKTNAPHNPVNTDFTRHTRQESDSTVRRFFNQVQ,10.282136779067205
+RPLVAIDFGTTYSGYAFSFKNQPETITLHWNSEISKALRKPTVLLIDSNMKEVAFGYEAENKFATLALDAEEKHFFFEKFKMALYDKNDRSILPSMRSANGTEKKAIDVFAEAIRYFKDHALKTINSTYPIDKQDLLWSVTVPSDWDARSKEFMRQAAVKAGLGEASLASEPEAASMYCVEHEVNKFGDEIKSGTKFLVVDVGGGTVDITVHEVLENNHLKELYKASGGPYGSVGIDQEFMKLFQLIVGAEAIEQFKIK,11.589466291126676
+MKVSVIIPTYNERENLEELFSRIDQALQGLNYEIVVVDDDSPDRTWEKAQELSSKYPIKVCRRTKEKGLSSAVIRGFKEASGDVFVVMDADLQHPPEVIPKLIEAIKNGSDIAIGSRYVKGARVENWPFYRKLISKGALVVTKIPLKDLKDMRDFACGFIAIKREVIEKIEFDENLTYGKILKILKYCWGGFSKVVEVPFTFGIRARGESKLKGKTIFEYLRHIWSLNYTFFRILKLIFALGFTFFGVSLAYLTLVLMEKYFLWYIPGWAN,12.090375297427133
+PGMQLNEFSSSGLGRAYSGEGAIADDAGNVSRNPALITMFDRPTFSAGAVYIDPDVNISGNSPLGAPGGTPSDREMKLVPTSHIALPINDRLAFGFAAYSNFGLATDYGDTFVGSTTPTDLEMKLNSLSIGGNAEITDQLSFGASITYQRAKIERFAGDLGQLVAGQIMQSPAGQTQQALLQAQSQGNLGSALAYANGIDSNTKIAHLNGNQWGYGWNAGILYELDKNNRYALTYRSEVKMTFKGNYSNDMPGYYEMNVPAWHNVSLYHE,12.173339409793382
+DASRVYYEDRSVVKEDGSVVKEGPFDLQSTLTLSGVVRDYASGTPLADAEITLTGPAFRAHTNSYGKFVFEGLAAGTYTLSVSRFGYEPVSETIAVSAGQTVESNVALFALASEVEILEVTADADPVFNTGDVATSVGTREMKEIPTVVGDVDVIKSLQLLPGVASAGEGTSGFYVRGGGIDQNLYLLDNIPVYNVGHLFGFFSTFNSDAIKDVTLYKGGVPARYGGRLSSVLDITMKEGNSDKLSGTASIGLLPASAKLQGPI,12.228122271950522
+GAVIDLSTATFDFGGSYTGVAVGDTITAVVTAPTEDDYVFQWFKDNVLQSGATGNSYTLTAAEAGKAIKVVVSGSKSGYTSTAKTAAVTTAITASSLTLTADKTKLTVGDTVTLTASLSDKNGNAVTGRTVKWSSSNTAVATVSSSGLVTGVAAGSATITASAEGQNGNGTANITVVAASVSSISLSPASASVAVGATQQFTASGYDSSGNVVTSGRVVTWASSNTSVATVSASGLVTAVAAGTATITVTSGGKSGNATVTVTAATLSSLSVSSSNL,12.23423450162324
+MQTYNNPEVTYDWWAGNARFANLSGLFIAAHVAQAALIMFWAGAFTLYEISWLTADQSMGEQGLILLPHLATLGLGVGDGGQVTDTFPFFVVGAVHLIASAVLGAGALFHTFRAPSDLAAASGAAKRFQNFNPDLSKLGFISRHTHAAKPELWSQLIGGKHKTTTGFAWVGVANPDGSITGMGTAGIQVKQAEGVTVGLAHYIWPLIGAAALAATICFFGYNSVITDIAYPEKKLEAVTFGYQTQAFDAFTQAGQVIGSTT,12.368396953842797
+AEGIRFAIVDEVDSILIDEARTPLIISGQAEDRTKELYKTLTRVLKSLEGGDYSVDLKNKKVSLTEKGVERTEKLLREAGIISDGTDNLYVVGAIFHAQKVATGKDYLFRKIVEKGRVEYTIDEKLKQVVIVDEFTGRMMPGRRYSDGLHQAIEAKEGVKVQRESKTLATITYQNYFRMFKKIMKLAGMTGTAETEAEEFKKIYNLDVVVIPTNEPMKRQDHSDQVYKTKREKYNAVLKEIEELYKKGQPVLVGTTSVEASEFLSNLLKKRKIPHNVLNAKPHAREAEIIAQAGRKG,12.697313288610662
+MPNFFIDRPIFAWVIAIIIMLAGGLAILKLPVAQYPTIAPPAVTISASYPGADAKTVQDTTVQVIEQAMNGVDNLMYMSSNSDSTGTATITLTFESGTDADIAQVQVQNKLQLAMPLLPQAVQQQQGVSVEKSSSSFLMVVGVINTDGTMTQEDISDYVAANMKDAISRTSGVGDVQLFGSQYAMLIRMKPDLLNKFGVTANDVISALQAQNSQVEAGSIGQLPTLPGTPLQLSITAQSQLSSEQEYGDIMLRVNQDGSRVLLRDVAKIELGGENYDIIAE,12.907199708267516
+DPLYYTNNGGLGFVLSALFGYIWWGYKSGTPKEVRSEAKYRMLTVVVPCYNEEKTIGRTLCSLLESDYPEDKLQIICVNDGSKDKTLKELEDFELRDVPLVVIDQENGGKARALNAGIDAASYEYFACVDADSQVEKDSLKKMVHHFADPSVGCVAGRVKIGNRWSWISRLIDLIQYLIAFNIGRRGINSITVVPGAIGAYRVSAIKKAGGFSGKTMTEDLDLTIAILRAGYKVVYEPEAICWTDVPETLKGFTRQRFRWTYGTMQ,12.993370901156627
+DISAEDRMWSDAEKRMEWQRIDRQVANRKSHGKRGLLSRIFGWIFRRNMDEKALKLLPHIKCYTPAEIANAIQSMTPEDLQRYELRASMFSLADKSNSGTISLTEFRNILECLGVQMSPTELQTLFQVCDRDQNDMINFNEFANRFHEPAKEIGFNVAVLLTNLSEHVPHDPRLRNFLELAESVLNYFQPFLGRIEIMGSAKRIERVYFEISESSRTQWEKPQVKESREFRTMQEIYNHIYYHTKQKENENVQRNAERWKMIEENKL,13.119829828981848
+SDITRLIVLVGTTLGVVLFLALAVWIVKSFWSPYQEINDWALALTIVDVLVVGVPAALPSTVTVTMALGAAYLAKKQALVKKLPIVESLSGVEILCSDKTGTLTKNKLSLQGAWLPGSEKPEQISGLVPEGSRQNITKCIHIAVLCNRASYKDGKLVGTPTEKAILKGLECWGVGYGEMRKKYPLVHQIPFNSTNKFQLSIHDKDNRYLLVMKGAPERVLEKCSTVLLQGKEQPLDEQWHTAFQTAYLSLGGLGERVLGFCQLYLSE,13.625918655212923
+MEVTLFALLALVVASAIIAWGPVTKPLHPHEALVDVGGHKMHYICQGKGSPTVILEAGGGGGSIEWGWVQPQVAAVSRVCTYDRAGYGWSDPAPHARDAGIVAEELHRLLRAAQVPGPYVLVGHSIGGFNTLHFAARYPQDVAGLVLVDATHEDQYRRWKGYEQEMAPFTSGQALDNLAANVRVMESLPPVDAGKVRDLPVLVLSAGREHPPFDMKLYREQWQREVVDLSNVSDRQKHIVADRSGHHIQFDEPDLVVAAIRE,14.117540370332351
+MDYHEDDKRFRREELCREAEFLKLKMPTKKVYHISETRGLLKTINSVLQKITDPIQPKVAEHRPQTTKRLSYPFSREKQHLFDLTDRDSFFDSKTRSTIVYEILKRTTCGITSLLANGIYWLAISTPTINEYPSFLSPSLYAAVLPFTFGFVVSFITLPRKALEYIEQNGQGKAAVHHHTHTHDHDAGDVKIVVNDKDLESHVVAGALMFVAALFSLVFHQWWSDYCDVAYTVFIRVRDVIFGHVKWT,14.986517088631075
+PSNISAWWNFGSLLGACLILQITTGLFLAMHYSPDTTTAFSSLSHICRDVNFGWFIRNLHANGASFFFICIFLHIGRGLYYGSYLYKETWNTGVILLLTVMATAFMGYVLPWGQMSFWGATVITSLAVYLPWWGQHVQKLLFQLIPALLVLLTAWTPFLIGYTLIRETTETESTNYGTPLRLHRIISHHLLLLRAVAXXXXXXXXXXXXXXXXXXEIKAAFWSVFHFILPFMATALAAPRSLLLDEANSTNTLVTTNLIFNFIFFLLPIFPATLSMFSPNLLGDPENFTPANPLVTPPHIKPEWYFLFAYAILRSIPNKLGGVLALAASVLILFLIPFLHKSKQRSTMTF,15.230674054330438
+SRTSELAVGIFVIIFGIALFFLAMKVSGLVGTNLSDGYTMKAQFDNVNGLKPRAKVTMSGVTIGRVDSITLDPVTRLATVTFDLDGKLTSFNAEQLKEVQKNALDELRYSSDYTQATPAQQMKACSEQMMTLLAPQQKEKKTLEVGDIIATSKSSVIYNDMSTYLNDLIGDLGTIASGVNELWPTLQANFSTVKTMAQNLLTANQQLPQLLGNVQTTSQLLAQDNNNFNKLVTDFALTIDALNAVVSKSGANLDTAIATANDLNTVLTENRQ,15.441068484289472
+DDVTVVYQNGLPVISVRLPSRRERCQFTLKPISDSVGVFLRQLQEEDRGIDRVAIYSPDGVRVAASTGIDLLLLDDQLIIREKYQIFINDMSPGAKVAQTAPAREIKWDHEALTEELTYEGQSEKLRDKDRTEVRRTMLNLERRLSDIRRQLAPLEKVRIEISRKMEDKTIQSYALWLMLAVVVCLMGLAWWQVLASLATFCVAVIIMVFVGRNWSAVLQRRRKRMGAEELRHRAYQTHQCHLCAICFTNQKMATLVPCGHVFCEECIKQHL,17.259236085949127
+DNTTNIVHVPVHYVFIMALPIIMCILGLLLNVLALWVFYGHMKRTTSVVYVINLAIADLLFVLSLPMYIHYYFNKTHWVFGELLCRITGTLFYMNTYCSILFLTCISIHRFLGVCYPFRLNLVKRNYAVCVSVGVWAFVMLACMPTLVFNQTEDYEGNRTICYDHLEDAQRHWALYLQVKVNVFVIGFLIPFLIITFCYSQIVATLLKVEANLAKKKSKAIRLVLTVVTVFVLSQFPYNFILLAKTIKLQQINSSCEFEKIIE,17.688518287684857
+MDYHEDDKRFRREELCREAEFLKLKMPTKKVYHISETRGLLKTINSVLQKITDPIQPKVAEHRPQTTKRLSYPFSREKQHLFDLTDRDSFFDSKTRSTIVYEILKRTTCGITSLLANGIYSAAYPLHDGDYEGDNVEFYGDYTIHAGDPENGGQCVVITLTDYGNYEPFYSASLEFSRKHFGFSALSVQCELSDVQSFTAVKQQFINLLSSRAPITVRKFVSPEFPRNSDSHDIFSLSCDVSNTGHVTAVTCQVSARFLTRYLTD,17.749448694031326
+MDNKLTLALAAIMVVLIAFVGINVMNNVNTNPTVVKTATVERGEYVERVDATGKVVAAQSTDLSFPATGEVTWLKVKVGDRVSKGQLLAELDTTDLEAQKNLALSQLEQSRASLALTRQTLARQQALAQTQAVSQQDLDNATNALRVQEAQLNQLRSGSRPEDIAAAQSQLRMAQDDLNRLRNGSRSEELRSLQAQLDVDKAKLNWDQKIVRRNQVRAPFAGVIAERLAEPGALVSPSQPILSLVADDNLEIEANVSEADILHLKPGQKAWFT,18.826645316378666
+MVSVIIPAYNEEKYLEKCLESVRNQTYKNLEIILVDDGSKDKTLEIAKEYAKKDERVKVVTQENGGVSSARNRGIEESKGEWIAFLDADDYWEENHLEELVKAIESNNCDMSICNAIWYYWWDENKRIIKRLPRESVIEAEDFFKELPIFMLTVVVWNKLFKKELFDSIRFPEGKTYEDTATIVDVLMKCKKVAYLNKALVNYRIREGSASTSFNPAKAKDHLKAIEVAFKEAHAEGLGDVALRAFQRRYVNSII,19.064942570982527
+MRPNLFLLALPFIALAAPAHAESITVNGDARIRALGKQNYAEVRTHISDNGTKATVDATGHLRIDAPLGERAQVKAYGELEAIYAKPSGDKNKASNTERLAYAGLKFADHGSIDYGRNYGILYDTNAWTDVFPLWGADVLESNTAAYKRTYGNVLTYRNNNAFGYVDGLSFALQYQGKNPTTGEVVKGDRVNSDGRRLGAATVGYDFDGFGIGFAAASSKTEQNGIKKDTDGREYAVAGSAKVGAAQVAGTYAETRNATRFGQTGKGRVE,19.328842227230165
+DDALPLSYYGTNKGLDRPATGPDRREHRFGFIADASAYPSQQLFIRGKVDVRDYQGSDTLRDDNAYVRLRNLTVGYDNLLPGSPLNVVAQFDLFNVLNATNVKDYQEVLSGGKAAAANFPIPRTYTLGLKLTF,21.699704462300236
+MAGRKILRDPYIIKLLELTEHNPGKRVTARCTSEGILTVPPDLICCLLIQLPIDSIDHHSFILNLQCKDDYQLILKNGSVLHSSCKYTPGKPAEVKAEGGSISIAITKLQLSDSGLYSCQPPNHEPSHGQLNLTVYKQTGFISVSDTGVGIVRVRAYAERPDDLNVTLTCLVTGVFPHDVTVQWTKNNSPLSKDSSPAEEQQHEDGTFFLYSKLTVDKSRWERGDTYTCVVAHEALPNKITKTLDRSKCQGEGLAPL,21.725914279351123
+MIMTMTLTMMMVMISNKTHWNSFQMNLMMTSLMILSLGGLPPLTGFLPKWIIITELMKNNNLITASMAMMALLNLFFYTRLIYSTSLMKLYPTNNQTKTKPKMMTHQMKLTALMTITMSSMTLPLAPQLITTELMAFAXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXTTLIMMATSLPIIMKPMTPWWC,22.24920166064802
+GVVKAAVSFCLAFCLVLCIAVTAVWFLSPTSDLDKKAVLPREYEQFKANQSSDQLRAFAAQYGLDATPAEGATDAMLAKGREIYVVNCSICHGSDARGASGLGVTLNPRPPSFTDPGFAAAHTDGEIFWVIRHGIRNSAMPAWKDKISEQDRWDLVHFLRTFKPESQKELTEAEIAALSVGEKVTMGQALFQEKCIVCHGANGQGNQTVGPVLNPSPRNFTSGVFKLRSTDQGELYAIRNGIRQHGMPPWGSQLKD,23.148315124172598
+MRTHTGEKPFACDVCDKRFNQKAHLNRHKTIHTGERPFACDICNKKFSEAGHMKIHTRTHTGERPFSCDVCSKKFSQKIHLNTHMRIHTGDKPFVCSVCNKSFSRKGDLNKHMITHTGLKPYSCDICSKSFSLKYNLITHKRIHTGEKPFVCDVCGKAFTQKCNLNIHMLIHAGVKPYSCDLCNMSFTQKSSLNTHLRTHAGVKPYACDLCGKSFALRQTLSRHHKTHTGVKAFACDFCDKTFFANQHLKRHRLTHS,24.00786399231911
+MRITKGFTLIELMIVVAIIGILAAFAVPAYNDYIARSQAAEGLTLADGLKVRISDHLEAGTCTADNTAVNGKTIGTEGTVGALPEGVSGDCKLSVAFTAGAAGKEITVKYDHKAGALTYQSATGKTISLVLPASLITKAGSWQGSVSWDYLKNLVPTNLRYAYVRSYMGPDYNPNNWPASGSTMPSDICWKSGDPNYTGTPGCTKNNSVAWGYPINPATCTFTPVADPTPTLAPVASVSLNKCYSAGTATLTATAA,24.29027038113483
+MVGRVGGWIVSVDPDGRFGPKPYKRHRAGIKDALSYLYQLKCRLRIDPDTWREWASPLRESITLEECRYTMPSFAVQASFMTLYWSVCEALFGCRFVYGPFNPILGETYEAHVADSDDEGQKTRYFAEQVSHHPPISACHVDSEKFYLDGHTCIRSKLTGKAISVHHVGQSCLTFKRAGETYLIHMPNQYCRSILTVPWAEQETVHCPTENHSAILEFTKGGFSAKFTGRWSSVLHVISAPHAATAEEKYPVTQVD,24.583101326751542
+GFHYFDITLAYFIPALLALLSSAWLIRAIRMDRADERAALTARIDELEQHNAALQARVDELERHVAMRTSELLETEQALAAERAALLDKGNHLASNFDTLKQRVAQLESERDALAADRDNLRGERDTLSGQVATLEAQRDEFARQLDAARQQAATAEERARQAEAAAASLRQRLDEALARVTELAGQNSELQAALARERQNNDALNARVRELEEQVARAQAGANQAQAARDRAQADAERLRQLEQQLAGANEAARRRIADLEDQLNRANRTIAEL,25.434296722653517
+QDTVADETGFFETELTVGTKEDRYSTVFNYRRINRDLKEPQDVNVYYARYEWQVAEDWKLRPGIRLDHDDFFGLTSSPKAYLMYEHDNGDTYKLGVARAYKAPNLYQSNPNYILYSKGQGCYGSSSCYLQGNGDLKAETSVNKELGVEYHHDRFAAALNLFHNDYKDKIVAGISTGVSGNSEMTTANYMEGWMTSVKWDWQIADNWKTDTSISWSRNKPKTSSSLDYQLRPENTLNSTLTWQARENLDFGWRVVHYG,26.433123728733182
+PPSECPPSPCGEKEYFDVCGQCCKKCKPMEGKISTACRKISDAVCDSGEWVEHPASDKCYACQKTCATRRPTQKACAAMRDCKCLDYFYRQLCVSCIPKCPRACDNQFCTAICNPGCVCPEGLFQDEFTGLCVPESECRTGCSNGQVYRECTSPCPSTCGNPNPRPSCSKTCFDGCACPEGMVLDDQNICVLPEQCGCTLYGRHYKPGETFTSDCGNPCEPTCENAYRTVVCTR,26.468925245048567
+ENKYSLLYKNQTLFDEWGIKYQVKSRMIEKSLYSVVFNVNDKKYNIIMRLYDKETKRIYSKREIINYIKNNSSINYKIDLIENGEYYAIAMPYIKGCTLRQYINKHISEKDFINILQPLIETLKVLHDKGIYHRDLKPENILIEQDENLFMIDLGLAIDLTNAIPTIDYGTDGFMAPEQALGNKPTFASDIYSLGVIAIELLTLKNPFDSNISLSESNWISTLHKKDKPLSSVLSKLILKMLEPSPNDRPNIKDVLNSLNSLEVLQRGVN,27.369412815985804
+EKKRKRDAVTWPPEKRQDAILFYLKNHNAPGMEFTEVAKAAGIHKSTVSRELKDPTFPPDASSRAGPGRPKKLSAKADELLNAWIKDTYVEGDLRREVTANILREKALEHGIIELSASTVWRILHKQLGYSSKKMSNRAIAADRRQVQEYRLEVIKAMHDNPYIYLDEIWINQNEAMNHVWFHDSETGLRSTMGLNKGSRGKRIIGVIDAEGFLHYEFKSTTDSTAAKTIVDFLEHNEGDNYLIVVDNAKYHSRL,29.60231093300791
+MVLFRATLVLTLFCVQLALAQVGINTSTPKATLDITAKTTDGSKPEGLLIPRVDRQQAETIPANPQLTIYTDGKTGKGFFYLGTTTPAGTANILDISKNGYYFYNGTAWVALNSGTYGSGTSGTPSATTDKEIYTNSTDKKVGFYSPTGTLVGYNSLTTTDYNSLITSGVTPSYAIGTSNTAALSSFYTGSVSGTLVTTGLTPVIGAAATNIYTVLDGGTSSTITIGSGGTVTSVTPIGGVTSVSLPLSGVSAVSITGSGSTITMGSGGVVTSVTAPSSVSTISITPTSGSIT,29.77388588380658
+LATLRQLWAGTFRRLWRAGDRDPDPAKVPLRARLVLMAALPVLALVLSAALTWQAASEQVRSATDRTLLGEVAEIGRTVSTAYGDVDTRLRGQLDGLARIPGVRSAAVVPLGAEGGTTVLGERTVPAADRSRWFSSLPLRSGSPDTVVSAPVLRGDRVLGSVQVVLDTDRVNALVSGLGWVLLLDWLAVTLLLWAAAMVLLRRQLRPLARMTAVAGAVAGGDLSRRVPDPGPDEVAQLGRAFNTMLDRIEQLLAGQRALLDDVSGELNARTVEL,30.911742604776983
+LSSSCFPWSLGVSVMTFISLSLLSYGPDRPLCPLTPTLSSLQFLVGTWKMVEGSGMFQEFCNHSASQWTFTADGHMTSKAFYVQPQQGQQLRCEEMRLIAQKHHPDTHRCRSLGQPPDTPYHYEYRRDCQDPLTMQHYVTEVMSRRLILSRQKPWDPAPDHIPPGTKIRYVSSPWGPEFCEPVPTQGEAVTLHGTVTHHTLGPLWGEGNHTALTDGFPEGVSPDVFLSAWGPKGLEKLNSLAR,32.019316129846914
+SPLQIVRDHFIREGRLIDPPEREFVPNDMPEYVLPSGERLPPIDVAKSRHRAVMPPPPSDYMAEYMAYADIMAPVTYYTRKDLGLGTKTILVAGAIGGLCGFLWFFMYVKGMGVLDALGITPFQIVRGDFSDTMSMANGFHMFFMITCGICFGGWATNWSRKAGFSDSMEASLMSAVVAYVLMVPMMMGATHTEMLANGHLMDLTHWTVAHLNPFHMMGFFAINVVAGLASIMVFALHLWYALTVRKTFDPEVELKTLKN,33.613951886997285
+MAYRSLFTSESVSEGHPDKIADQISDAVLDAVIAADQASCGTAKAAVTTGLVTIGGESAMCWVMSDMIRTTLVDIGYSVTAVGDEGGFAPNIQSFHDALKVIGDTIVNTRKAQSDTNVQIGIDVCATSAKVLPTEYMGYEDRGASLIFSHRSGETEDSTIADFCVGVLAADIKQTLPPIVAELGKPARLRAMGQLAPLAEDAAFVGYDWNHTTGFPRFSAGSMSTADALAAADNTADAAAMANTALAEAAMAGDHATAARWSAAVEDLTAQAKAGTITTGKIAEAIRAACL,34.430385572117466
+SPDVQIHPPKRDPDPWGIKGLSAFLLGGATLWGLAALAIHLAGLVPFPTVELGTADFHMTLPFMAAAAGGFLIAKHQPRDMFGIGMPEDRPLIATGAAVSFALVVIALVLYAVAPGTYTPRAIGLVGSLAVSAGILGVFGAVLGRLRPVRGIGLVPAAILEGIARQPEARGPVLVSMVAGFALGAVGLLAPHHFGLAFGFGAIGGLGAVALAGWTGALVGAPDISGPTAIAAKMQRFYLWATVLPVAAMVVALVAIATPHLNLGIGEGLLLGGMLAGPLCVAA,35.89969391101693
+SAWNTNLNMDARSAWATYQRQNGEVIGWMPIVNYADTIHDRDFAQAQLIFSTQVSKLWWAEDLGVNAFVVTLSNDLYQLWLNSPDEKADLMKQININAYNINWGVDDGTYADFQVWNIARMLRNDPSTNGKRYFAYGSDAPLIAAYRDQGWETNTVRGYGEYVVLPKAAGTVDNEVAQAAVDNWYSGAIANRLGTMANTGAVVQTGTTDNGIYGYAMTDGKTLYFPRYNTKYYNTDQGGVAHEFGHHVDYAV,39.68959897551875
+GEKWIMKFDGALNPSNISAVLAGGLIGLAVGLQATFFNVSTTSHVTGVLGGATVVGMATYYKWASPWAISAGTFFSLVLGTYLGSQLVKRLHVYKLPEPIAFFGGSFVMVWLWSWMTTYIYPASHALTPYASHLSYLCAMLLGALGGILGSLITPPLKDTFIASALGIIGGTGFAVSHLTMLNPTIPSTLYAIAYAATGIWGAITATRIARVLNLFEGALVCGAATVFYSFVKVVAPELLPVALASIVCAAGVLYVANLTKVV,45.78566418659657
+LAPSPKVFFIDDTPIQWGFVIILLLLSSGGLFFDSKLAGIFTSLGIAVGLIGAALTTFADTRKGKVTPEQLDRVNSTLKTFFGWSLISGVLGLAIYAASLNIDGKLAFVDSLFYFTGTGLVTVGFGDIVPTTTAAKILVVVLIVGGIGFAGSMISTVASWIRSQQEKSELDKHTIRAHARNIVICHDDPRVSALCEYLQGYFLVDDKQSTYHVLPMYLDGNSLERRALRKKLFSNRVAKHFAREGSVRDLDAVRRANVAGARAVIVLSKADENID,47.50012378184719
+GSTDLSTWQTYVQSTAATITSYYQDTASQAQKNQVLANVTQIINQLDSSTKTKAEVDSALTAINKIKAQIAGDAGGGSSTQATIQGVLDNLITKANNLLRQGQTISEVNALISDLNNLVTQAKGQARSDQESVYTKADSALSNLQNQLNQEREVGSNDRYISQTEKDNLIQNVNNYINNEYLWTDGTSNEGQRLTAAKNLISDTLTNDQKRAAQDAINQLIKDANDLLNQARDRAANQGVTQTEKDNAISNVKTVY,51.616257412346954
+PVPVPVPVRRPSNTQLDSPGHLRTLLDRNHLPPPDTQLSPDNRLLQDNVPGSGRPLPERTRLSPDRRTLQDFPVHGRDLPEVHRDHGLPAPDHIPPGYGGFLTEAQRHKEWFHVSDTHMAPPDGTSYPIARFHVSAGRPGMPAPDRYFAALGGAQGMASHMHGSGMHSSHGMHGSMGMHGMGHGMFGGGAMGPVFFIVAALAIIIAIGVAVAAKAGGGEGA,52.07059943074766
+APFAICRRCRRRRGLPVCARRRWRRRRGNIWCAVGSGGIWRPCCRCITRITCRLRVSAAWRICCAGCRGRTCCGSFWWSTTCGSRACTARWTPSPWRSTGRCTRGAWTRWWRCARSTATPSPATASTWRAGWRAATPCGSATSTTSSCRCSTAW,52.5490203150644
+LVLFAPTFNLSDPEGTVFATLVAIATAVGGYVAIPISGIDSIAGGVVSGYAVAKAGQFTNALKTTAMGAAVGEILGEQLYFGGFGPLGIVAGLITAGAIHKWLVMNKVSVNIYDAIGGRRFEVVLAVMIVTGLIMSFFVPAPVGGFIDNAVSKVGQSAAIGFITDSGSTLLANGINPVIAIGFLFAMAGVLIGGFKVASAQMGTLMGAVAFITGAFGFAVHFGANMVGVGALIAGRFTGRAFSDKVNETWPAVTDAVNNRYRTMVNVLAGSVVGAIFGL,52.5972401908542
+MAAIHPPNLSFLPKPSAIHLFAFWTGSMGCLCPLLLGSQPILWASTALLLGTLQLGMGLKASLYPSPFPSHHLFQTTNYFLSFFLPFSLLSYASFFPSTLFPPGAIVTLTGLTLHGVSAYTLGGATGAWINYNTNHIFTAENGTVTGIKEMDTYSMVTANRFWSQVFQILFWCTNALALATHFSRIWTISRAEKHQLHVEEEHHHTAAEMVLAENIGIKTLTDYDDDDKMISYYRKDGVHHMHVEDAELALKLQEEEDLKNKKN,52.98995197391265
+GLFAVIVEIPFSLRLPSVVQAKGSFSDSLFSHSAYPVVQPYFSPETLFGFDILLPITGEPVSRGLYTGHQPLLVVGVETSFLLTVETRLTGEVYSKGGRNSWDIQNCNFFGSDGKKYSLPAFERKKVKDVKCVDQDGVFSEVILERTHTSFTLKYTLPDSEWLIHSRSQLVKREDSNMGRPRKHLSSLVARNSSFEATYQRVSEKETEVSVQFGFSVGWKVIYLFLVKHFPFVFHWISNVLFYLLLNTLFAYIPDFSTFDCLAFLVTL,53.005620188296234
+LTPRQRMWYGILSTAVFLLASEGSFFAISLTALVSYAYYQSILAQTQPAAAPAISAGFAFMLGVVIFGWVVLGVIQALINAISEWIRALVINIYSRTVFAPYVRALSHTPEGVRVINLQSSQLAGLFVNEFVKGFVDGLALIASLLVSLLISLWMGGVLGLIVFLYFCFRVMRQVGENMGRLREAQGQMYEQTLGLVEGLKDIRAARREEVYKGRIESLFGELAGMEVAGAKVQAVSTLMMRVVTQVAYLCMLWVGAYGVFHGDLS,56.40990415587325
+MNINQLVLKAREENKQHENFQQGRLNLRYQEISKIEYLNRCRKLAINGNRIQRINDLQFFYHLTYLDLSNNLITSIENLHCLPLLRNLNLQKNLIGHITGLETLVRLEYLNLSHNQISKLENLECLVNLERLDVSHNHLTKIEGVCFLKSNILKELNLESNLLQELKFCEHLDYVTISNNNISSFSQVCYLLEHMPRLKYLSFTGNPYEQKLKQYRMVVFSKLQYLDGFVITEEELCRGSEVVDWIDSGSEFQRFRYCVINFLKDENNRT,58.18325968813114
+GWVRQLPVYKRFAPFLSKFTLVTSLIAVGAGSGATYIQNLRKPRVRDKIVVHTVPLTPEMSGGKRFSVAPPSGIPHASHRMIPIERQREEDAXRERALRKKMLRRTAMLASGAFCLVLFVALGATIGTLRSEGVLKKDEFIPRPAIVGADGKAYDMDHPYAPPVKYQVQWEPKMGEKYYFHDYAKHHPNDNPENPYNKVAARA,60.55516244953947
+MSASLFQTQGNYLVAAAISLSGLFMLVGLLAGSPRRPTYRWLLASVTLFCVAVSYFFMLSATTLEQGLVVKTNTGERALVDAVNGSVQYADGHYEIEATLRNLGSQPVRVEISRLQVVGEKMFGDIQSRTVEVGPNETRQVKFLLNRVLTSSANFRDRVLFVITDAQGNRQFIEVPVAYQYAQITGLLIALAWLAVIVIGFPVAWRSRMRIASGNRPVASGPQIAYLTALLFAATWTLVLMIAGTQIIGSQAGL,60.895889009456674
+AQTTLNVADNSGARQLMCIRVIGASGNCSFVNQQKCTGICGCTRNATPIESEEIFDCIMKCGGQPGDCEVFQTHQCQQRMANNAHHYRRHWLSHTDFCVLPEHFHLDQDRHFHFQQHHHNWHHGHRHHHHHHDFHFGKFFETFAAPFASIFGGHIHGGFEKFSEMLANGFGGFDMFFGGFGGHGHFGGYEQEATSFKILASVVAAILLIAIAIPLGWLVKSQVSGIKVITTTTSGANQIILMKTVVAIATILAIAIAIPTG,62.10915337352313
+GKRKAAVSRAAKLAATRAVPFARAAAIGPYAAIAIAGTKMAIDDHYKKDREKNREFVFNQWMSRKQLYDYKRKFWMFGPEKMKQLYEESGAKGAEAFFKENAETFKKIRDEYLVDLKNGTANPLTGEKVPLNPALPEDIRFPKYTPPPGLVPEGENPYYIPPPGYVREAERAGMPPPKKREMRMRPAGSEPGTTFGGAGYNPFAADPEYPHTAYAXXXXXXXXXXXXXXXXXXXXXXXXXXXXKRKAALSRTARLVATRAIPFGRSAAIGPYAMTAVAVAKMAYKDD,62.61105811999597
+GWVRQLPAEERPVVLDRDEIELDPPVIGMGRNLAIMAVSVFLFMILTAWFALGEIQESEIARGTLRADRTLLDRTFIPITERGVFTTLDSRWALADVEPGELVWIAVDKHPATLQPGQSVQVYVRAVNDKPDNSVITPYRAVFAEIEREGFRWIVSVDQERFDQFRAHVTESLRLVNRGEALVGADGAPIPTIDLEATPGLAPDIPVTLRFEAEDIDWRILDQSQVQVARANVASADVSQPGWQEVELTAVAPWQAGKT,66.68297956107664
+GWTLHPVSLYFSNHLGYVRLYQLWLTSVDKKSTNAFYHEVSDSQRKLVKRITRMELCFLGVMTLISLASLAIYAKFDQTSLPMLNKVFPRQNDIVTPVKFSLSASFFVFFLLLACFLSHAVNQVAKLASFCSALEDIQEFYVRIREELDSLRSYVENLEKRSAVSEEKLRLQASQTEMLLKRLPSFSSFCLLTLDRPILLSSHCPSLLPTVKGILNRGYKLSVYDPPPFQLGLCKDTHISDTQIYYNNGSRLEGATFHL,71.44388492712908
+ALVPSDVSDQAEATLAFARQNLAKIEPEKIEIKQEPASGVNPADQPSQLDIYLTCTLKNEIRAPPGTTMPQLNFLRNQLEKNLLVPASQRDAYIQANPQQTLILDQPSPLTPEQKEDLAQLTITYGKNNLEVNTQRWPLPSLQVAMQTLESGEAHLEYRIHALPKAAGQPPVPVLKLVSKTTLPATAPVPNTASPTLSVRLPPRRTPPPPPIADEDLDDSPIVRDSRTLLKILLPTVLALVIALIAWRLWSSFTSHRIEAIATVPLPSATATPTP,72.99576740691371
+MATPSFVSEPFAGLTPRQRQAIAAAMRSSLGYVQESVALNRMYSSALQGLVKPAGNAATIVASTGNVGTALSTLSGIQTAFSQYLKGKGSLVGSATNTLIAAQGKLISDLGSLITQEKTFMDSISKKLISDMDIAVSRTQTINSEVTKLTQERNALVAQLEAARKDADSAQKATITTELSNIIGTVAGAFLTAGFTAGIVLSIWELWAWGATLAAIAVGVGILLIIYATSRSSASNRKAELDAANSNLQNAQSTLKSDQQ,76.45022543295501
+TPGLIDKLLGGGVQLPPGLLMALAVLAIQLGFIALIGKRVQFGAVARRYKIDAPETSLITAVLLGLAGYLAIFFAMRGMPWSATGELRWISGPHLNPLTFTAKFAMCALIVVPAAMRGWWAFSGPGADERSRHNARYAFWGSIVAVTALVVEGFLIMAPSLTEARFSPFYYARLLTYFVVTTALLVWTTVRESETPGRTLMGFALFSAAMVGLEMLSFTRFAVQFPTWWNVEVANLMYFGTMMIVLGLFFAMGGNIRWMVAA,77.96717901193621
+MRYFKIRSTTLLIYLAAISVCALSICAPGFITPDEPAHFNYIRYLADHGQLPRIDPYAYASWGSTLSSLSYEFFAALFSWIPLETARSTVIFFAILNAVIIFATARRIAARYGSTGAFAAAAVFLLSPRVLAQSSFNNYDSLGIALMLAAWIFYEKVLTEKRLLPAVLSAVAVSIALLTNYQGYFIFAAVLLFSLPFPKLFFSRKNILFSAGVLSAAVIAAGLFAVFYKDLFLYSVFDVRLMSVFKMMTHQYPFSDAMTIYGGYFTVLF,81.23947975232642
+LSPDLVAQLKAKTGVSYKEAKEALEATNGDIVAATIWLHEQARTSTFFFFFFFFNLVVGMGLFGPDRPLWLPGHALRLQPRHGLPGHRAAGLHRPRALPRLRLRPLPRLPRLRQGRPHLRHAPARPLRQPRLLPGDHHGRLLPRPRRLPLRLRLRHPRLLLRHLRLRPVLLPRRPLRALRLPRQPRRPLLPLLRRLLQGHPHRGARLPLRPVRRHLLHLRVRPLLRRLLGLLRARVRPQLLRPAPQRLHRLPRAQARPQGPRDGLLLPGQGPAQG,81.88453571822619
+VGRINTAVTKVEGLKGVFDTASVFIIMRLILGALPGHDYFWHVATHKVLSTTWYELFSNVFLQVPSFITTFFMGAMLVQTMAQKSPEMQEFLKKGGIIFMTLAWFFFAPSGDYVVMRVISACTALVFIVTSMLEMNHVTPPPDTGLPRPIALCLRAFFYIGFLEWCVQQNFYAMCVLFFFMLGGVFTHYTALFVARYMKFFETFVPPIVHSGFSIAWMMWATQEGFITPMGQEPLLLTVLSVMVFFSVMSMC,86.07420147138896
+MVYRITTIILVISTLTSFLIMFIPLTFRTFHYVMAFMVLLETSMFMWWYFDMSTSSYWNQERVHYEENGVPEFSLSFWSGLMFQMASVCYTYGKVYLSALRFGDMDHVQGQFIDLSNHFAMKTGLNPNDFKMRWPIQLMHNIINTMVEETEKLNAKQQREGITAEVEGEGRPQTFYEIQMLWHCITIILDELKRCTTVSNAIMTKETVDRMVHLCEKGIIPPDLEDFVFKLVFFTPPFEMILNFAI,92.19736060379442
+MFSKLSLDAVPFARAPQWQRHLLRVACLISLFSLAYLAIVIAADTTNSIFTVGIGILLAAGVWFYWRDAVREELSHNPLGTRAAGIILGSGLVMLGLQLSAHLTGTWGYVTPTTFRWLAIMALAWPAAFLALRLTRDEEPVSEAMDNFDRAMAIMLVVSLVLWTFSPLLRGAVQHLHWLLFADYCFVVVDVVAVVMIYHMVRFLLAPLRETHPDAAQAIARKADAMVLWLFLWALYPIAYLVPAFVWGFHFPEGSIW,94.53532369154783
+MVIVAIDRAIKATTILISPLIVIDLISAFIIGFRYQLVHDVIARICFAMVFVYYFALFFEIYYSRHFQGYQSALIKRCFLTLVPWLIYGPLLLLYRPVGDWYFPMTLLAFTIFALLAKRFVIEEETRDVMLEKERRMHFFAMVLFVGAMAIAFALSHFGVLEAFMPYRAFWMRGVTLIYFTSFYLVLLHHYGLREEIAFHKRGEVKPYPAYLAYTVINLTAWAVFFFFTHYAPTSAFARWWAWANFICIPFYAIG,95.44574656037318
+MLLVFFAVMAGLLYGWWLRGSPKHARVTLGFYFITFFLLALLVWTHLGPSQAWSGFSVTLNRFYFWYLIITANAGAVLSAFGLVHRKTYVPEAERKRISLQFDAVFLILWLASALVCTFVMCEYLRWGWTGTDTLFGNHYLTPVLGPLLFWEWVTGLGLVVFAVLCWIYVRKFHYHDNLTARFAYSLLFVAPLIYLWMWVAGHPYQLAWTQDTAWLQSMGYWNGYPFMNPAHMIAFLGAGALFSLAMVAHGFRSERDGY,99.45883530953378
+FGKDVVPVAATMVPFFGAIGFVLALRQPHFYPPAILIHGFIAAHFIGLYGENDFGEDFVPYFVAGLFVFWGFVAFILNVYFPPTPQNKRTLREEKYHEQVSVLTQAAIDGQEPQEIEVALGQVQANFDTAKSALEADRLIANQKLRAAVSTAATLIVMTAVIVGVHSQYDLLGLVLAMAAAISTLAGLYVFVGVSRAVLTFFTLRRGKTDEFLADADNFLKRNPVPVAALSALAKGHRDQAVAAAQSAIDNINPNPTTSSSSSTSASSSSSWAFDPLG,100.05271078832499
+MRCGGTMPSTRSTTTGRCTRGWRRRICGTGTATSWTRPTGCTSRSRTSTTSWAGSPTCRTSRTGSSTPWTCPCTCTWRTRSSRSSCRTSPSWTSRRSTRSSRCTKTCWWTRSSRTTWCISGRCSWRSGSTCTRALSWSTWGRCATRRPTWSARSTSPTRRWSSACTARSGRRWCSTTPTSCTWGSCTPARTTGAWKPCSGSTSSPTPTSRPQARCSSATWSRACPTWRSTSSTRRRTSSAATPSSGPW,101.66820225573242
+RIHDTILPFLMLGVGAFLSGIATLIEKSPNIMKCLPLLLTIGCCIPFLGWVSPIVLPFFSMKTQTTLSDGAIYGNSSISRVYENGIVEETQYVCGLNIFTSRIEVDGDFLFPKYYAPTNDTELQYVTEIPTSAHGTNPAELNATRKNLLNTLGPRYTLVLTDTDGVVRDYVVGNIPQGSPSPNLRYKGLRLELAVDQLPAYTISPPDGTSAFTFINKHWLIDIPTTLISETMVRKLVKAAGPLGPAYIVITEQSPNPIVATAGQAR,103.31987729217545
+CRFGTCTVQKLAHQIYQFTDKDKDNVAPRSKISPQGYVNPNNEPTSYISPGHLRTKKSNMIPAKEVTRIDPNIVPNPNVQYPNLPAPYMYSGRAKRRRNLGLLMGRPNENPDNQHEMQDGYESAAYSNSYKGTYGKLTRWTSRWINNHYIDIERKVHFKDGRIFKTRAESSRINPKIGDFKTTKYITRGEKEALGFKIGGRLLLRPSSKLEKNFTVTETRTIRNGYTTTISRTIRWDDLEKCPLGNCAVGDLVTIDVTD,103.52201921223038
+MWWWRLLVVALLRIGLALEDPARNPCSRVFFEGLTGCQQKVLRAVYPDPSRCLKACSEMKEAANSWGTRYAVATSVLGLEWLAYSWIQDKVACRCRGLSIPPTQKPSLFEKLLHSPLLLQGLQRAAEPVLGFFTQATQALKEAVWSALQWLGGQAGHILAFSRHFAFCLMAFSTLTLLSVCCNWWAIRRRFHQLESVTEQLLRCQQYVLQFRAVSRRHYISWALQLYFAHAFILRACAQLVSVLTTVSNMVSDSFSL,113.06700569292313
+DPLSIILGILAGLFLIILIVLYFCGPYCTCIKRTGCCGNRWCYRWRCCCRRRRWRRRCCRWRTCYRYRWYSTRVRKCVKVPVVKTYKYRSKCGTCYGYVTRTRKVCCSYSSSSKKVCYTACRKKVYKTRYTYKVKVRNCKPCTKYRTKTVCSKCSYKTKIRTRTYKVRVAKCPRKSYKVVTMCKKKPSYRTCSRTSRRSRKVCLTCGSKAYRTKRTIKVPVKKTCSRKVSYKVCARTSHRTH,118.16791808043538
+MQTYNNPEVTYFDRSQTDVEYGWWSGNSAWKNEQWLVMTKEEAKEFFRRSWIKLLDAFLPTTYIIVRWYQMYNYGCPLWCEKNDGKAHCKDWDYHPTCGKGPWWANNPLPTVKGQCEVYTTHRAGSSKECRSYYDLTSAQKAALQSSDCKATTGVYPFYSQAGTCRLNADYPLEKIPEGICNVHLNHKTRASHWGDPDIPTSQIWYFAAYDQAEKEWRTLSGTLEHTWVALSHEDYQRLVDIESKVPWSVSP,119.47290840438525
+VQHFTGYVEDGRGIFYSLPMTNKGLDRIMLCIAVIVAFGMLLCPLASLYFSSEPVLVREDIFSALRTLSIFAAVWQIADVLRRTYVVVSKNPLLLLGLALELTFYTVYFGLDKLYPYPLAVFLPLQFGGILLRHITSIYLQAVSSRNESIIAQLRREREREARRTRERNIAQKRRIDAALWRQMSAVVIFLLLWLIAFTSSALALYNNLLASQQLSIAGLTPSQAASLNTGELLLRVIYGLVISCSAVLFTMTVEARDKIMHD,126.04829832003558
+MLARFFRRQRTASFSLATVVALSALALHTSGMHRPALYASATAVHAITLITLGVMYARSMAPRAEGADHDLRHFMTAYLVTALAWPLAMVLTFALTHFLPGTDPLVPDRTLRLVTLINLAFMASATAHFAFALHTGWNVPRAIAATIVVFALVALTAWLIEIFTGGSTHWSFWAVLIASAAIWLGLALYFRRHAAAIAAFERRHNAQILARFIAAQDETHEQAGGGARSLAHNLDSPLTAAALFADDLSGKVDAPVREHLRLIRRSAND,146.856677830673
+GWFDAMLASVSEFAPIFVVLIIFIVRVYKPFGSEWIVHVLHIADKRPGLNALIHRLLPRTAVHVPQAVKDKYVFLNSEHCIQFGCRHDPVPQYLELLSKGTYSLKVDVWFKHDRAREFYNMLLNEAQTASENHHASKIRHWTDEKMSELFAMAKKAYIPLNETREHSHDKAHSHNHAHSHNHSHDHGHSHEHTHGHDGVHAHDHSNTSDAHVHSHKHLHLHVHVHDKKNIIRRMNSALRKMKAAGVNTHEVAHVHDETTP,166.7331315912073
+GAPPITGEALEKDISRREKGVGGFLSRLFSLVNSTNPFAVGVEGEKLLEEIENIRDSMGHQTAAQLYFAQQQSLLQAEYARWQESHNATLQATKDHIFNAQLGHILMLAGAVVCYTAGLRAWAXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXFLAITNTAINTYFGGSITQSLEQVSAAVEHAKKVGLLSQETVGQIEAAYNSATGKALSYNNIADTEAAFQEFSRNHVLRSQLDKENADAAWTRVQSEGASLRAYLDHASRAITSALNGAIFAFGGP,169.6224292611664
+MSAMSVQIDRLQDQLNHLGELVAQNSKVIAALTQRIQVLERIVTERLRIPYIPLEKRTALMFPLHDEEKQSEITLFINAELHLGTAPGKHKVYYTTVEEMIQHFKEGKCLPQNWPQSDNPFWPCYRELADEMKSNTAAYNNFIKMQDEMRKLCIMLSSGVYHISRNPGGAKDLYTDPKLFIQIYTNECLRNAIPAEILDQMIIDLYANYTEADIHNMAEVRASRNFNHLEKQYMHKLLKLKKTLPFAIQASMDVVL,187.47480920419864
+AAGVAAWLPFARAAAIGWMPVASTAPRAMTATASWPIWMIWAMAMPAMTGRRWRRSRWATAPMARSSGAAPMMARPVMTAMPVAAATIRPSALRSISATASATAGSASAMTRSAAMSPIRTIWRRCSRARRAVASGAWSAISATSSARRRTTSTARALSCAAMASGLPSASMKAAAGGGSSNTMPRCSGSSASRTRACGTASPSCCRAATAASASARAARARSCRASSRARSAIWRAMSVRSRTWSRSARLRRSTSRPSMRSATAA,216.8557544805602
+EKKEVCSVFLTNRVPLDDKRFRRERVYLPGESPFIDPDLFLSREHPLRAQVRGTIIEWLRASIYGIYPYPEQRDPNLWCTERFKQEVMPDGHCEPTLGFVPLTFSTCLTRDMIAASSYNWRKTMEVPGAKMLLHVGPLGTGGHYDYAFTFLQPDNTFAYVKGNKLVRQTKIWNDAGFQLVTEEATLLDAQEYFGAANKLGVCIFCGNCVEYCPTNCLSMCEEVLPRGNALQESWTILERVFMPEDPEHENFKYRRLRTSDGAKFINYTS,520.388790480398

benchmarks/Generation/ProtGPT2/run_clm.py

@@ -0,0 +1,657 @@
+#!/usr/bin/env python
+# coding=utf-8
+# Copyright 2020 The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Fine-tuning the library models for causal language modeling (GPT, GPT-2, CTRL, ...) on a text file or a dataset.
+
+Here is the full list of checkpoints on the hub that can be fine-tuned by this script:
+https://huggingface.co/models?filter=text-generation
+"""
+# You can also adapt this script on your own causal language modeling task. Pointers for this are left as comments.
+
+import logging
+import math
+import os
+import sys
+from dataclasses import dataclass, field
+from itertools import chain
+from typing import Optional
+
+import datasets
+import evaluate
+import torch
+from datasets import load_dataset
+
+import transformers
+from transformers import (
+    CONFIG_MAPPING,
+    MODEL_FOR_CAUSAL_LM_MAPPING,
+    AutoConfig,
+    AutoModelForCausalLM,
+    AutoTokenizer,
+    HfArgumentParser,
+    Trainer,
+    TrainingArguments,
+    default_data_collator,
+    is_torch_xla_available,
+    set_seed,
+)
+from transformers.testing_utils import CaptureLogger
+from transformers.trainer_utils import get_last_checkpoint
+from transformers.utils import check_min_version, send_example_telemetry
+from transformers.utils.versions import require_version
+
+
+# Will error if the minimal version of Transformers is not installed. Remove at your own risks.
+check_min_version("4.45.0.dev0")
+
+require_version("datasets>=2.14.0", "To fix: pip install -r examples/pytorch/language-modeling/requirements.txt")
+
+logger = logging.getLogger(__name__)
+
+
+MODEL_CONFIG_CLASSES = list(MODEL_FOR_CAUSAL_LM_MAPPING.keys())
+MODEL_TYPES = tuple(conf.model_type for conf in MODEL_CONFIG_CLASSES)
+
+
+@dataclass
+class ModelArguments:
+    """
+    Arguments pertaining to which model/config/tokenizer we are going to fine-tune, or train from scratch.
+    """
+
+    model_name_or_path: Optional[str] = field(
+        default=None,
+        metadata={
+            "help": (
+                "The model checkpoint for weights initialization. Don't set if you want to train a model from scratch."
+            )
+        },
+    )
+    model_type: Optional[str] = field(
+        default=None,
+        metadata={"help": "If training from scratch, pass a model type from the list: " + ", ".join(MODEL_TYPES)},
+    )
+    config_overrides: Optional[str] = field(
+        default=None,
+        metadata={
+            "help": (
+                "Override some existing default config settings when a model is trained from scratch. Example: "
+                "n_embd=10,resid_pdrop=0.2,scale_attn_weights=false,summary_type=cls_index"
+            )
+        },
+    )
+    config_name: Optional[str] = field(
+        default=None, metadata={"help": "Pretrained config name or path if not the same as model_name"}
+    )
+    tokenizer_name: Optional[str] = field(
+        default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"}
+    )
+    cache_dir: Optional[str] = field(
+        default=None,
+        metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"},
+    )
+    use_fast_tokenizer: bool = field(
+        default=True,
+        metadata={"help": "Whether to use one of the fast tokenizer (backed by the tokenizers library) or not."},
+    )
+    model_revision: str = field(
+        default="main",
+        metadata={"help": "The specific model version to use (can be a branch name, tag name or commit id)."},
+    )
+    token: str = field(
+        default=None,
+        metadata={
+            "help": (
+                "The token to use as HTTP bearer authorization for remote files. If not specified, will use the token "
+                "generated when running `huggingface-cli login` (stored in `~/.huggingface`)."
+            )
+        },
+    )
+    trust_remote_code: bool = field(
+        default=False,
+        metadata={
+            "help": (
+                "Whether to trust the execution of code from datasets/models defined on the Hub."
+                " This option should only be set to `True` for repositories you trust and in which you have read the"
+                " code, as it will execute code present on the Hub on your local machine."
+            )
+        },
+    )
+    torch_dtype: Optional[str] = field(
+        default=None,
+        metadata={
+            "help": (
+                "Override the default `torch.dtype` and load the model under this dtype. If `auto` is passed, the "
+                "dtype will be automatically derived from the model's weights."
+            ),
+            "choices": ["auto", "bfloat16", "float16", "float32"],
+        },
+    )
+    low_cpu_mem_usage: bool = field(
+        default=False,
+        metadata={
+            "help": (
+                "It is an option to create the model as an empty shell, then only materialize its parameters when the pretrained weights are loaded. "
+                "set True will benefit LLM loading time and RAM consumption."
+            )
+        },
+    )
+
+    def __post_init__(self):
+        if self.config_overrides is not None and (self.config_name is not None or self.model_name_or_path is not None):
+            raise ValueError(
+                "--config_overrides can't be used in combination with --config_name or --model_name_or_path"
+            )
+
+
+@dataclass
+class DataTrainingArguments:
+    """
+    Arguments pertaining to what data we are going to input our model for training and eval.
+    """
+
+    dataset_name: Optional[str] = field(
+        default=None, metadata={"help": "The name of the dataset to use (via the datasets library)."}
+    )
+    dataset_config_name: Optional[str] = field(
+        default=None, metadata={"help": "The configuration name of the dataset to use (via the datasets library)."}
+    )
+    train_file: Optional[str] = field(default=None, metadata={"help": "The input training data file (a text file)."})
+    validation_file: Optional[str] = field(
+        default=None,
+        metadata={"help": "An optional input evaluation data file to evaluate the perplexity on (a text file)."},
+    )
+    max_train_samples: Optional[int] = field(
+        default=None,
+        metadata={
+            "help": (
+                "For debugging purposes or quicker training, truncate the number of training examples to this "
+                "value if set."
+            )
+        },
+    )
+    max_eval_samples: Optional[int] = field(
+        default=None,
+        metadata={
+            "help": (
+                "For debugging purposes or quicker training, truncate the number of evaluation examples to this "
+                "value if set."
+            )
+        },
+    )
+    streaming: bool = field(default=False, metadata={"help": "Enable streaming mode"})
+    block_size: Optional[int] = field(
+        default=None,
+        metadata={
+            "help": (
+                "Optional input sequence length after tokenization. "
+                "The training dataset will be truncated in block of this size for training. "
+                "Default to the model max input length for single sentence inputs (take into account special tokens)."
+            )
+        },
+    )
+    overwrite_cache: bool = field(
+        default=False, metadata={"help": "Overwrite the cached training and evaluation sets"}
+    )
+    validation_split_percentage: Optional[int] = field(
+        default=5,
+        metadata={
+            "help": "The percentage of the train set used as validation set in case there's no validation split"
+        },
+    )
+    preprocessing_num_workers: Optional[int] = field(
+        default=None,
+        metadata={"help": "The number of processes to use for the preprocessing."},
+    )
+    keep_linebreaks: bool = field(
+        default=True, metadata={"help": "Whether to keep line breaks when using TXT files or not."}
+    )
+
+    def __post_init__(self):
+        if self.streaming:
+            require_version("datasets>=2.0.0", "The streaming feature requires `datasets>=2.0.0`")
+
+        if self.dataset_name is None and self.train_file is None and self.validation_file is None:
+            raise ValueError("Need either a dataset name or a training/validation file.")
+        else:
+            if self.train_file is not None:
+                extension = self.train_file.split(".")[-1]
+                assert extension in ["csv", "json", "txt"], "`train_file` should be a csv, a json or a txt file."
+            if self.validation_file is not None:
+                extension = self.validation_file.split(".")[-1]
+                assert extension in ["csv", "json", "txt"], "`validation_file` should be a csv, a json or a txt file."
+
+
+def main():
+    # See all possible arguments in src/transformers/training_args.py
+    # or by passing the --help flag to this script.
+    # We now keep distinct sets of args, for a cleaner separation of concerns.
+
+    parser = HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments))
+    if len(sys.argv) == 2 and sys.argv[1].endswith(".json"):
+        # If we pass only one argument to the script and it's the path to a json file,
+        # let's parse it to get our arguments.
+        model_args, data_args, training_args = parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]))
+    else:
+        model_args, data_args, training_args = parser.parse_args_into_dataclasses()
+
+    # Sending telemetry. Tracking the example usage helps us better allocate resources to maintain them. The
+    # information sent is the one passed as arguments along with your Python/PyTorch versions.
+    send_example_telemetry("run_clm", model_args, data_args)
+
+    # Setup logging
+    logging.basicConfig(
+        format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
+        datefmt="%m/%d/%Y %H:%M:%S",
+        handlers=[logging.StreamHandler(sys.stdout)],
+    )
+
+    if training_args.should_log:
+        # The default of training_args.log_level is passive, so we set log level at info here to have that default.
+        transformers.utils.logging.set_verbosity_info()
+
+    log_level = training_args.get_process_log_level()
+    logger.setLevel(log_level)
+    datasets.utils.logging.set_verbosity(log_level)
+    transformers.utils.logging.set_verbosity(log_level)
+    transformers.utils.logging.enable_default_handler()
+    transformers.utils.logging.enable_explicit_format()
+
+    # Log on each process the small summary:
+    logger.warning(
+        f"Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}, "
+        + f"distributed training: {training_args.parallel_mode.value == 'distributed'}, 16-bits training: {training_args.fp16}"
+    )
+    logger.info(f"Training/evaluation parameters {training_args}")
+
+    # Detecting last checkpoint.
+    last_checkpoint = None
+    if os.path.isdir(training_args.output_dir) and training_args.do_train and not training_args.overwrite_output_dir:
+        last_checkpoint = get_last_checkpoint(training_args.output_dir)
+        if last_checkpoint is None and len(os.listdir(training_args.output_dir)) > 0:
+            raise ValueError(
+                f"Output directory ({training_args.output_dir}) already exists and is not empty. "
+                "Use --overwrite_output_dir to overcome."
+            )
+        elif last_checkpoint is not None and training_args.resume_from_checkpoint is None:
+            logger.info(
+                f"Checkpoint detected, resuming training at {last_checkpoint}. To avoid this behavior, change "
+                "the `--output_dir` or add `--overwrite_output_dir` to train from scratch."
+            )
+
+    # Set seed before initializing model.
+    set_seed(training_args.seed)
+
+    # Get the datasets: you can either provide your own CSV/JSON/TXT training and evaluation files (see below)
+    # or just provide the name of one of the public datasets available on the hub at https://huggingface.co/datasets/
+    # (the dataset will be downloaded automatically from the datasets Hub).
+    #
+    # For CSV/JSON files, this script will use the column called 'text' or the first column if no column called
+    # 'text' is found. You can easily tweak this behavior (see below).
+    #
+    # In distributed training, the load_dataset function guarantee that only one local process can concurrently
+    # download the dataset.
+    if data_args.dataset_name is not None:
+        # Downloading and loading a dataset from the hub.
+        raw_datasets = load_dataset(
+            data_args.dataset_name,
+            data_args.dataset_config_name,
+            cache_dir=model_args.cache_dir,
+            token=model_args.token,
+            streaming=data_args.streaming,
+            trust_remote_code=model_args.trust_remote_code,
+        )
+        if "validation" not in raw_datasets.keys():
+            raw_datasets["validation"] = load_dataset(
+                data_args.dataset_name,
+                data_args.dataset_config_name,
+                split=f"train[:{data_args.validation_split_percentage}%]",
+                cache_dir=model_args.cache_dir,
+                token=model_args.token,
+                streaming=data_args.streaming,
+                trust_remote_code=model_args.trust_remote_code,
+            )
+            raw_datasets["train"] = load_dataset(
+                data_args.dataset_name,
+                data_args.dataset_config_name,
+                split=f"train[{data_args.validation_split_percentage}%:]",
+                cache_dir=model_args.cache_dir,
+                token=model_args.token,
+                streaming=data_args.streaming,
+                trust_remote_code=model_args.trust_remote_code,
+            )
+    else:
+        data_files = {}
+        dataset_args = {}
+        if data_args.train_file is not None:
+            data_files["train"] = data_args.train_file
+        if data_args.validation_file is not None:
+            data_files["validation"] = data_args.validation_file
+        extension = (
+            data_args.train_file.split(".")[-1]
+            if data_args.train_file is not None
+            else data_args.validation_file.split(".")[-1]
+        )
+        if extension == "txt":
+            extension = "text"
+            dataset_args["keep_linebreaks"] = data_args.keep_linebreaks
+        raw_datasets = load_dataset(
+            extension,
+            data_files=data_files,
+            cache_dir=model_args.cache_dir,
+            token=model_args.token,
+            **dataset_args,
+        )
+        # If no validation data is there, validation_split_percentage will be used to divide the dataset.
+        if "validation" not in raw_datasets.keys():
+            raw_datasets["validation"] = load_dataset(
+                extension,
+                data_files=data_files,
+                split=f"train[:{data_args.validation_split_percentage}%]",
+                cache_dir=model_args.cache_dir,
+                token=model_args.token,
+                **dataset_args,
+            )
+            raw_datasets["train"] = load_dataset(
+                extension,
+                data_files=data_files,
+                split=f"train[{data_args.validation_split_percentage}%:]",
+                cache_dir=model_args.cache_dir,
+                token=model_args.token,
+                **dataset_args,
+            )
+
+    # See more about loading any type of standard or custom dataset (from files, python dict, pandas DataFrame, etc) at
+    # https://huggingface.co/docs/datasets/loading_datasets.
+
+    # Load pretrained model and tokenizer
+    #
+    # Distributed training:
+    # The .from_pretrained methods guarantee that only one local process can concurrently
+    # download model & vocab.
+
+    config_kwargs = {
+        "cache_dir": model_args.cache_dir,
+        "revision": model_args.model_revision,
+        "token": model_args.token,
+        "trust_remote_code": model_args.trust_remote_code,
+    }
+    if model_args.config_name:
+        config = AutoConfig.from_pretrained(model_args.config_name, **config_kwargs)
+    elif model_args.model_name_or_path:
+        config = AutoConfig.from_pretrained(model_args.model_name_or_path, **config_kwargs)
+    else:
+        config = CONFIG_MAPPING[model_args.model_type]()
+        logger.warning("You are instantiating a new config instance from scratch.")
+        if model_args.config_overrides is not None:
+            logger.info(f"Overriding config: {model_args.config_overrides}")
+            config.update_from_string(model_args.config_overrides)
+            logger.info(f"New config: {config}")
+
+    tokenizer_kwargs = {
+        "cache_dir": model_args.cache_dir,
+        "use_fast": model_args.use_fast_tokenizer,
+        "revision": model_args.model_revision,
+        "token": model_args.token,
+        "trust_remote_code": model_args.trust_remote_code,
+    }
+    if model_args.tokenizer_name:
+        tokenizer = AutoTokenizer.from_pretrained(model_args.tokenizer_name, **tokenizer_kwargs)
+    elif model_args.model_name_or_path:
+        tokenizer = AutoTokenizer.from_pretrained(model_args.model_name_or_path, **tokenizer_kwargs)
+    else:
+        raise ValueError(
+            "You are instantiating a new tokenizer from scratch. This is not supported by this script. "
+            "You can do it from another script, save it, and load it from here, using --tokenizer_name."
+        )
+
+    if model_args.model_name_or_path:
+        torch_dtype = (
+            model_args.torch_dtype
+            if model_args.torch_dtype in ["auto", None]
+            else getattr(torch, model_args.torch_dtype)
+        )
+        model = AutoModelForCausalLM.from_pretrained(
+            model_args.model_name_or_path,
+            from_tf=bool(".ckpt" in model_args.model_name_or_path),
+            config=config,
+            cache_dir=model_args.cache_dir,
+            revision=model_args.model_revision,
+            token=model_args.token,
+            trust_remote_code=model_args.trust_remote_code,
+            torch_dtype=torch_dtype,
+            low_cpu_mem_usage=model_args.low_cpu_mem_usage,
+        )
+    else:
+        model = AutoModelForCausalLM.from_config(config, trust_remote_code=model_args.trust_remote_code)
+        n_params = sum({p.data_ptr(): p.numel() for p in model.parameters()}.values())
+        logger.info(f"Training new model from scratch - Total size={n_params/2**20:.2f}M params")
+
+    # We resize the embeddings only when necessary to avoid index errors. If you are creating a model from scratch
+    # on a small vocab and want a smaller embedding size, remove this test.
+    embedding_size = model.get_input_embeddings().weight.shape[0]
+    if len(tokenizer) > embedding_size:
+        model.resize_token_embeddings(len(tokenizer))
+
+    # Preprocessing the datasets.
+    # First we tokenize all the texts.
+    if training_args.do_train:
+        column_names = list(raw_datasets["train"].features)
+    else:
+        column_names = list(raw_datasets["validation"].features)
+    text_column_name = "text" if "text" in column_names else column_names[0]
+
+    # since this will be pickled to avoid _LazyModule error in Hasher force logger loading before tokenize_function
+    tok_logger = transformers.utils.logging.get_logger("transformers.tokenization_utils_base")
+
+    def tokenize_function(examples):
+        with CaptureLogger(tok_logger) as cl:
+            output = tokenizer(examples[text_column_name])
+        # clm input could be much much longer than block_size
+        if "Token indices sequence length is longer than the" in cl.out:
+            tok_logger.warning(
+                "^^^^^^^^^^^^^^^^ Please ignore the warning above - this long input will be chunked into smaller bits"
+                " before being passed to the model."
+            )
+        return output
+
+    with training_args.main_process_first(desc="dataset map tokenization"):
+        if not data_args.streaming:
+            tokenized_datasets = raw_datasets.map(
+                tokenize_function,
+                batched=True,
+                num_proc=data_args.preprocessing_num_workers,
+                remove_columns=column_names,
+                load_from_cache_file=not data_args.overwrite_cache,
+                desc="Running tokenizer on dataset",
+            )
+        else:
+            tokenized_datasets = raw_datasets.map(
+                tokenize_function,
+                batched=True,
+                remove_columns=column_names,
+            )
+    if hasattr(config, "max_position_embeddings"):
+        max_pos_embeddings = config.max_position_embeddings
+    else:
+        # Define a default value if the attribute is missing in the config.
+        max_pos_embeddings = 1024
+
+    if data_args.block_size is None:
+        block_size = tokenizer.model_max_length
+        if block_size > max_pos_embeddings:
+            logger.warning(
+                f"The tokenizer picked seems to have a very large `model_max_length` ({tokenizer.model_max_length}). "
+                f"Using block_size={min(1024, max_pos_embeddings)} instead. You can change that default value by passing --block_size xxx."
+            )
+            if max_pos_embeddings > 0:
+                block_size = min(1024, max_pos_embeddings)
+            else:
+                block_size = 1024
+    else:
+        if data_args.block_size > tokenizer.model_max_length:
+            logger.warning(
+                f"The block_size passed ({data_args.block_size}) is larger than the maximum length for the model "
+                f"({tokenizer.model_max_length}). Using block_size={tokenizer.model_max_length}."
+            )
+        block_size = min(data_args.block_size, tokenizer.model_max_length)
+
+    # Main data processing function that will concatenate all texts from our dataset and generate chunks of block_size.
+    def group_texts(examples):
+        # Concatenate all texts.
+        concatenated_examples = {k: list(chain(*examples[k])) for k in examples.keys()}
+        total_length = len(concatenated_examples[list(examples.keys())[0]])
+        # We drop the small remainder, and if the total_length < block_size  we exclude this batch and return an empty dict.
+        # We could add padding if the model supported it instead of this drop, you can customize this part to your needs.
+        total_length = (total_length // block_size) * block_size
+        # Split by chunks of max_len.
+        result = {
+            k: [t[i : i + block_size] for i in range(0, total_length, block_size)]
+            for k, t in concatenated_examples.items()
+        }
+        result["labels"] = result["input_ids"].copy()
+        return result
+
+    # Note that with `batched=True`, this map processes 1,000 texts together, so group_texts throws away a remainder
+    # for each of those groups of 1,000 texts. You can adjust that batch_size here but a higher value might be slower
+    # to preprocess.
+    #
+    # To speed up this part, we use multiprocessing. See the documentation of the map method for more information:
+    # https://huggingface.co/docs/datasets/process#map
+
+    with training_args.main_process_first(desc="grouping texts together"):
+        if not data_args.streaming:
+            lm_datasets = tokenized_datasets.map(
+                group_texts,
+                batched=True,
+                num_proc=data_args.preprocessing_num_workers,
+                load_from_cache_file=not data_args.overwrite_cache,
+                desc=f"Grouping texts in chunks of {block_size}",
+            )
+        else:
+            lm_datasets = tokenized_datasets.map(
+                group_texts,
+                batched=True,
+            )
+
+    if training_args.do_train:
+        if "train" not in tokenized_datasets:
+            raise ValueError("--do_train requires a train dataset")
+        train_dataset = lm_datasets["train"]
+        if data_args.max_train_samples is not None:
+            max_train_samples = min(len(train_dataset), data_args.max_train_samples)
+            train_dataset = train_dataset.select(range(max_train_samples))
+
+    if training_args.do_eval:
+        if "validation" not in tokenized_datasets:
+            raise ValueError("--do_eval requires a validation dataset")
+        eval_dataset = lm_datasets["validation"]
+        if data_args.max_eval_samples is not None:
+            max_eval_samples = min(len(eval_dataset), data_args.max_eval_samples)
+            eval_dataset = eval_dataset.select(range(max_eval_samples))
+
+        def preprocess_logits_for_metrics(logits, labels):
+            if isinstance(logits, tuple):
+                # Depending on the model and config, logits may contain extra tensors,
+                # like past_key_values, but logits always come first
+                logits = logits[0]
+            return logits.argmax(dim=-1)
+
+        metric = evaluate.load("accuracy", cache_dir=model_args.cache_dir)
+
+        def compute_metrics(eval_preds):
+            preds, labels = eval_preds
+            # preds have the same shape as the labels, after the argmax(-1) has been calculated
+            # by preprocess_logits_for_metrics but we need to shift the labels
+            labels = labels[:, 1:].reshape(-1)
+            preds = preds[:, :-1].reshape(-1)
+            return metric.compute(predictions=preds, references=labels)
+
+    # Initialize our Trainer
+    trainer = Trainer(
+        model=model,
+        args=training_args,
+        train_dataset=train_dataset if training_args.do_train else None,
+        eval_dataset=eval_dataset if training_args.do_eval else None,
+        tokenizer=tokenizer,
+        # Data collator will default to DataCollatorWithPadding, so we change it.
+        data_collator=default_data_collator,
+        compute_metrics=compute_metrics if training_args.do_eval and not is_torch_xla_available() else None,
+        preprocess_logits_for_metrics=preprocess_logits_for_metrics
+        if training_args.do_eval and not is_torch_xla_available()
+        else None,
+    )
+
+    # Training
+    if training_args.do_train:
+        checkpoint = None
+        if training_args.resume_from_checkpoint is not None:
+            checkpoint = training_args.resume_from_checkpoint
+        elif last_checkpoint is not None:
+            checkpoint = last_checkpoint
+        train_result = trainer.train(resume_from_checkpoint=checkpoint)
+        trainer.save_model()  # Saves the tokenizer too for easy upload
+
+        metrics = train_result.metrics
+
+        max_train_samples = (
+            data_args.max_train_samples if data_args.max_train_samples is not None else len(train_dataset)
+        )
+        metrics["train_samples"] = min(max_train_samples, len(train_dataset))
+
+        trainer.log_metrics("train", metrics)
+        trainer.save_metrics("train", metrics)
+        trainer.save_state()
+
+    # Evaluation
+    if training_args.do_eval:
+        logger.info("*** Evaluate ***")
+
+        metrics = trainer.evaluate()
+
+        max_eval_samples = data_args.max_eval_samples if data_args.max_eval_samples is not None else len(eval_dataset)
+        metrics["eval_samples"] = min(max_eval_samples, len(eval_dataset))
+        try:
+            perplexity = math.exp(metrics["eval_loss"])
+        except OverflowError:
+            perplexity = float("inf")
+        metrics["perplexity"] = perplexity
+
+        trainer.log_metrics("eval", metrics)
+        trainer.save_metrics("eval", metrics)
+
+    kwargs = {"finetuned_from": model_args.model_name_or_path, "tasks": "text-generation"}
+    if data_args.dataset_name is not None:
+        kwargs["dataset_tags"] = data_args.dataset_name
+        if data_args.dataset_config_name is not None:
+            kwargs["dataset_args"] = data_args.dataset_config_name
+            kwargs["dataset"] = f"{data_args.dataset_name} {data_args.dataset_config_name}"
+        else:
+            kwargs["dataset"] = data_args.dataset_name
+
+    if training_args.push_to_hub:
+        trainer.push_to_hub(**kwargs)
+    else:
+        trainer.create_model_card(**kwargs)
+
+
+def _mp_fn(index):
+    # For xla_spawn (TPUs)
+    main()
+
+
+if __name__ == "__main__":
+    main()

benchmarks/Generation/Visualize/analyze_mdlm_denovo_gen.py

@@ -0,0 +1,7 @@
+import pandas as pd
+
+path = "/home/sg666/MDpLM/benchmarks/Generation"
+
+res = pd.read_csv(path + "/mdlm_de-novo_generation_results.csv")
+average_ppl = res['Perplexity'].mean()
+print(average_ppl)

benchmarks/Generation/Visualize/esm_umap.py

@@ -0,0 +1,111 @@
+import torch
+import pandas as pd
+import seaborn as sns
+import matplotlib.pyplot as plt
+from umap import UMAP
+from sklearn.manifold import TSNE
+from sklearn.decomposition import PCA
+from transformers import AutoModel, AutoTokenizer
+
+path = "/workspace/sg666/MDpLM/benchmarks/Generation"
+device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+esm_model_path = "facebook/esm2_t33_650M_UR50D"
+
+# Loads ESM model and tokenizer to embed the sequences
+def load_esm2_model(model_name):
+    tokenizer = AutoTokenizer.from_pretrained(model_name)
+    model = AutoModel.from_pretrained(model_name).to(device)
+    return tokenizer, model
+
+def get_latents(model, tokenizer, sequence):
+    inputs = tokenizer(sequence, return_tensors="pt").to(device)
+    with torch.no_grad():
+        outputs = model(**inputs)
+    embeddings = outputs.last_hidden_state.mean(dim=1).squeeze(0).cpu().numpy().tolist()
+    return embeddings
+
+# Load a random set of 100 human and reviewed sequences from uniprot
+def parse_fasta_file(file_path):
+    with open(file_path, 'r') as file:
+        lines = file.readlines()
+
+    sequences = []
+    current_seq = []
+    current_type = "UniProt"
+    
+    for line in lines:
+        line = line.strip()
+        if line.startswith('>'):
+            if current_seq:
+                sequences.append(("".join(current_seq), current_type))
+                current_seq = []
+        else:
+            current_seq.append(line)
+    if current_seq:
+        sequences.append(("".join(current_seq), current_type))
+    
+    return pd.DataFrame(sequences, columns=["Sequence", "Sequence Source"]).sample(100).reset_index(drop=True)
+
+
+# Obtain/clean sequences generated from ProtGPT2 fine-tuned on membrane sequences
+protgpt2_sequences = pd.read_csv(path + "/ProtGPT2/protgpt2_generated_sequences.csv")
+protgpt2_sequences['Sequence'] = protgpt2_sequences['Sequence'].str.replace('<|ENDOFTEXT|>', '', regex=False)
+protgpt2_sequences['Sequence'] = protgpt2_sequences['Sequence'].str.replace('""', '', regex=False)
+protgpt2_sequences['Sequence'] = protgpt2_sequences['Sequence'].str.replace('\n', '', regex=False)
+protgpt2_sequences['Sequence'] = protgpt2_sequences['Sequence'].str.replace('X', 'G', regex=False)
+protgpt2_sequences.drop(columns=['Perplexity'], inplace=True)
+protgpt2_sequences['Sequence Source'] = "ProtGPT2"
+bad_sequences = []
+for seq in protgpt2_sequences['Sequence']:
+    for residue in seq:
+        if residue in ['B', 'U', 'Z', 'O']:
+            bad_sequences.append(seq)
+protgpt2_sequences = protgpt2_sequences[~protgpt2_sequences['Sequence'].isin(bad_sequences)]
+
+
+# Load MDpLM generated sequences
+memdlm_sequences = pd.read_csv(path + "/mdlm_de-novo_generation_results.csv")
+memdlm_sequences.rename(columns={"Generated Sequence": "Sequence"}, inplace=True)
+memdlm_sequences.drop(columns=['Perplexity'], inplace=True)
+memdlm_sequences['Sequence Source'] = "MeMDLM"
+memdlm_sequences.reset_index(drop=True, inplace=True)
+
+# Load UniProt sequences
+# fasta_file_path = path + "/uniprot_human_and_reviewed.fasta"
+# other_sequences = parse_fasta_file(fasta_file_path)
+
+# Load test set sequences
+other_sequences = pd.read_csv("/workspace/sg666/MDpLM/data/membrane/test.csv")
+other_sequences['Sequence Source'] = "Test Set"
+other_sequences = other_sequences.sample(100)
+
+# Combine all sequences
+data = pd.concat([memdlm_sequences, protgpt2_sequences, other_sequences])
+
+
+# Load ESM model and tokenizer for embeddings
+tokenizer, model = load_esm2_model(esm_model_path)
+model = model.to(device)
+
+
+# Embed the sequences
+data['Embeddings'] = data['Sequence'].apply(lambda sequence: get_latents(model, tokenizer, sequence))
+data = data.reset_index(drop=True)
+umap_df = pd.DataFrame(data['Embeddings'].tolist())
+umap_df.index = data['Sequence Source']
+
+
+# Do PCA
+umap = UMAP(n_components=2)
+umap_features = umap.fit_transform(umap_df)
+umap_df['UMAP1'] = umap_features[:, 0]
+umap_df['UMAP2'] = umap_features[:, 1]
+
+# Visualize the PCA
+plt.figure(figsize=(8, 5),dpi=300)
+sns.scatterplot(x='UMAP1', y='UMAP2', hue='Sequence Source', data=umap_df, palette=['#297272', '#ff7477', "#9A77D0"], s=100)
+plt.xlabel('UMAP1')
+plt.ylabel('UMAP2')
+plt.title(f'ESM-650M Embeddings of Membrane Protein Sequences')
+plt.savefig('esm_umap.png')
+plt.show()

benchmarks/Generation/Visualize/mdlm_de-novo_generation_results.csv

@@ -0,0 +1,101 @@
+Generated Sequence,Perplexity
+GEGQPTLDAEGMPKADEGKMMTFKSENFTDDSVENLVLTSYGVYNPVIFTDLVIRTPKEGAVVPPTVVLMNGEWTEVMPNLTGAETFDTQSKYLVNGLKRYGVSKKKHVQVYQMARRTKDLLTMIPDGMASADFSFEAPGRANTMPAVGLSMDSAVGQPNLSRLRGVDVFFRYIVYTADPFGSETQNLEVQASERTNILFLNQQKKKVKSGIVVQMQKGILFERFGEVMDGQRPSNQRVGSQDMLIGVGALVKLNQKKIRTRIIQLFNLGYDDSEAIDWLPTTVAYLDSTYYVAMTTIQSIWVTDYYGLQGLFPFNQNKIGKHGVEVKHVQYFLEFVEAYVDQLEDLFTEYNERNSKLSNSNAIQAITIAEYQQLKDQLQLLTTENPIVDSSMIALRIKKLDNSATRELVSQFNRDVERATPNITAAQISVLKDNMTILLQDELMHMSDLNGEAADATYTLQAARESLEQLTTAAEFAPEYLTIEEQDISDFKARMELLKEIVGSLSNRIESAVKNKQDKEGIQYAMYKRPNRIDILIKNINLKFKGIQFQIDSIVAKVRNMEAFIKALVYRLDNVRISLVQRVGNRRHLAKKEKEPETVLIVNLRDYRSTLILFDIMTNLRITDEGQPENILRMKPVLDNADIPTENERIPSLSMPLMVRYTTVVINLPELDEHKAPLGINIVVAKDAVVSRLEWEWEGDVFKNKPYRIKRAGYGPDYVRAGALAQVFIARSDTATQSIAVRKTANEKFLLRLPRLPGSLMGEVVLKSFATFHQAFGTGRNNVYQRDEDSDKKYNQTLIDYWFDLNRFFGLSQREEGVQMMLLVEEPFTAGILSKAIVFDDDKKSAFLMMARAFLVYLPLHHSPDAPLEVANNSPKNIRLNLQATIAARG,18.2131
+WTTGWGVSQDLIDSASMSPGMIWILLVDSYKERWFGTYWWGTSTCKEGAFPFEDVMQRIELRILKKYFYYLAIISSVLTLLMIIAKLVTNCLSFANIHKSHRYFFCVNCFWFISQLCNDLSAFPVLKKLESATRFVIYPSPVKAVQLDTMPDKIVLYLIFLNIFSTHTVLVFQSMSLGLITGIIDIPTAKRIIVPNLGILVIKTFSSKNCKLSLLAPEMWPKCMYDYVAFKNIEAQIVITSTSVGAVLCLLLILKGSVFVSSSYMFVGGKPANPGTGTRMLLPKDDHFEHKFCHNFSNVEKISASSYAASPEESILLLVNKEEHNKLRVLAVVPKGARNVLVIEIMKLKPFQTTYNDLYLPRDENNQLQKNKKVVSVGKIVLKDPASWVYLPQGRLKMNFKKAYIKSGAAPILLSFGQRLISVDNAVPLAKMRTTGITVLEMAPRGSRVQAIVVLPGQLKCGKSETVYWFTVSSIDNNRRGIAKYMGGVTYRGRAFIDMDKNLAGPPLVSDAYQMLFNDWLEMLCGAMKATESEKVKSRKGASELRVIHRSHHGCIVAILDDLYRLRFDLVDIERIGMINEEGRINGKIRSFEFQNFMLTSKNDMKTGFVNMPESFKPRTILTGDLIDNDWAPSFDLAAIRHGNIQVLVDGNDLEGSEEATNCHHGNAFSLGPQGRKVVVGAVVAPKTATPKCNSISINLQELPANYVVLGAKALTAQHVANFSVNGTKHDKKTCKQHEEMWKMYIQKSGQISKQASIKQCAILLSLGKLRATVKWFLGKYLERIFHVLLKGCKTVETIIDKGRMHKIKLVKFRFGIWIFLSVVCLAELFCIMIFVAPEIVRGLHTLILFLLFMMLLLLNYADTGHEVTGVPYY,18.1991
+GIISVIDLGAKLIVPGDLFFVLCGKNRPPTSGALQYTILHTKKKLFCCGPTHAKHICLINGECIRDGREKLQNLCKTSGKWTEVRRPKSTYSCSLVYRRVQRRFCGPQKARAPVWILYIVLLSAIGVIIAVTINWVLQVCIILGAVVANGFLIRVLSIVDTRNQIITRGLRRYGIYRNSVKVACTSGSVVIVRIKFMEDQISGGWRPASFIRTSFKEFLPASATAFSRLADCNEKLIEALV,17.785
+RARILNRSLESARKYLDFLKIDKVEFYENEMTFRVFIEEAPEFKRMEITEDKIRLRLKPDKIRKFVELGNLFKYTDALQLAVQLEKQNKEELVIAEPEVIHALHKHHNHMPIFHLLEAFNDETVAEIILANIGKMPAFLFWVWNRMSDPTEDRGKGFNEKVKRKNPTIVSILDKQVYTIKGGFGAALTKSILGPLLATQNIKIKADYDESLADVFVGLHFVDGSIILRPWPITGKEVALSEEVTATDKVISASEVGSEEDKFQLTSIENNFTKTLLEIKKRRENAFEGAYTETGSKVSDKPVRELKANLKLIPEYQAERIDQTYWRKILRLSRSLISEPRGARAYLLRIGERVDPHRYRPAIGAEMLMGIPNSITTGFKISKTLGLQAAGLDLIQTFRSLSIRRMITKNFMAILIDKPGLKAVFWFIPLVPFIAVQLLIYGVLVGRAKPGNLVEIIDSMVDGKFETRPNGSPAASHKMVIGVKYSLIYPGNQAKKISLVRWNTALSKDVRGQAKEIDLWQLISYYLEKMRIGPSAVSNVFQSVHDGLKRNELAVLLIMDPKTRDDSMILDIMNLRIERGFVSLIKSYIHDYDEEVYMTYREILNQNVFLMKYEEYATMSADLEAYWLKSIEETNLRALRLPGAMRKQLFLANLCRISEHLDTPTEQDAFSNPEGITIDEGFTDEARAQGIGFVAGFVDEKEFEQRYALLAKVYIASLKALAAALADSGVKTGIKIGLGTVERIEVHKDGMMIDHVKMEGPGRFPIIVGGEVSPIVNGATIIPSFIKILADGKVDEGKSPNKTPTEKGEITPQSLYRGMGKTVVLNDNGGLQAHALTWAINDEYSYFVAMGTSNKSDEKQLAALSNSLDMTTFEDSAGRLFSSIVKAKTLSENGVITENGAEFL,17.7595
+DIMLPKSPLFEEMATLGFIGHTLLAPIKPWTKATATMVGITGIGVTIYWGVPDMFPFSPTNHWWVKGMKAIVPSIIALQIIDLFYVVLTGLTSRFVYPIVATFYDHYFVNVQILVTGIACTLVYPETHGDVVSVDLLQCRTDGKLTQIPMEEALELINFIDQIMEKTKCKFNISEGYYEVLMTKKFIVKGGKGVGPDTDPEPCEKYATWGRLRDPTGPIRPNRAEKSQIAIYAVCGGAVLQKLGVPLLDNEAPIPSQLLIFAIGTVGIAAITIVALIFGGIDIAMSNPVYRPILAYSPHNKLLYPLSTWDVGYYNVPNVTSTYVVVVVPPALTIYSVANEAIKVETTPIPVKFAEILPTGETALLSSYPLTIAQTDLNARYERREADIFTKHEGMQQTFKGEVVPLVSNNRLKSPSGVQIGCAASLMTVPEEDGMTPPRIATTWFEGPYSAPASDPLMRMPGDFYGYGKGTMDGTVSNEMNGISHRPVTLATGGNVKFSPVMLPWYIGARYGLDIQHSGNRRMAHWLTSRAVMGFFKKNVARLADRVNAITLQVPSKDPDLNNRPNHAVMSRQWITAVEAIKELAICSLLNQFREGLGHKNDTIEADLNPFSGVIAQSSMAILKTAMGATRSSVAQLSMGMALEAFRHQMTGTAGIHYLMSVTGNGPGRISTALNKLDSPLPPAIAED,18.2088
+MASLAAANKRRSITSIAVFLGIAAAVVVLGISGKVTDKNTVLDFSYGKNRGFKPQHLSEYVRSRYPAVVIAKGENLLRGGRFLPWVRPPGLRYVAYYDAIESWDIEDSLKELVDLRAQGLYADVETAAALKKASVSRSTITEMVGMHTNQSEPQFRTEQTKVAKSNGTMFVFATMSFTRGRYEIGSLMLSAIGKRWVEEPTPDTKMRYMKLRPSIQLLCRTGGIYRPIFQLGGPEGIFYHDGYGENSYQVDDFIWKHLERRA,17.9845
+VYGLLTTGSASSAMATTTLMIGVLFGLFSSFALPLWSWLQYLTTTGITTATGARYFKNIMIEAFSSYSAAIIGTVSIVPLGSSIPAAASASAVGAFGAITGFALGIYSMLFKKMDSFTHWLFSAGAGLSAGLAGLISGIGAIQIGNAATATSGTAIPLFGLVLRIIVSSVQGLLGTIAAILLISN,14.5579
+IGHVFHLLHMALPIWRPPLPVTPGEPHPRPIADLVTPAFEYKTLLRCPHPHVSPIFLSVVLWMIVALVLAGVMVAQALPAPTGARLLACYP,17.2759
+SIIFIFMYLTNCLQWRQNGHLQMSGLLFATLGVSTVTHTMMLASRGLCQAQKQRIRRECTLFGLAVHFCLAVGLFIASVSFAVWSSLEGLDDDANTVAVMRWWGWTFSFERYATVKVLFDQGIQSTYIMSWLLLMTKREDFRLLLFFMLTMFASILVPFSRGAHFYSLSVAFSNFATVILPGVNGVGNEIVQQIIFVLLFTFPMFLLIVDIAFFVNLIFKAAHP,17.039
+LPDVLYHYEERKFVITRSEVVLEPNELFIGKIAVDVANYNIKVKVDLRISSKYVVFNNSQLTWNDHFLQPLISDRLRWVIFRVCVGTSSPQLLIHIDMIENFLQQLLSFLKGIVVQQFVGKVTLIQEDMKIEEETALLIEIETYPEAELLKLVRNLIIKVEDRTIGSIRHEAQLAKINDWSAKRIISDLNIGDIDNGEHVLVSAQEELESSIMERLAGHLRKFVNVGTWTESAAIDIVARIYGSLSVELHEEWLVMLEYLFTYPNDYFPGMYTVQYYQNADPGELLKNHALIIDELQRLELYEG,17.8596
+SQGLDDLIMTTVADSEKDTDLTTTADLNMVPIYVGSNETATSQLGMIVKRKRPEKPIYVPVHSCSKDDRACAFVNFFNLARDLGYIEQDEQRVTPDWRAIILTMAEDHIHLWSHPNVAILKLRVIGEKLADQRETMKDPLNTRVEQVALVQAPKIDLIRASYGSLLEYQGEQKKYRINNTLSRCESLCADAGVGYMVATDAKVCQVEGETVDNNTGKDGDKTEILRAHFKQPSAFNKGSGVLRGHIMMTLGIGLLILLLYVIIVFFLHKIQNATFKFRIPRVAIATSLVADACMSVLAAGIAYALANFPVFTAKIYAETAVVLVLLVKGRLFIGKNKEIPMTARITIIRLAGAVLGFAATIIGIVILDPVLSIDGVSLAPGKVPQLLCLAQTAVREGGHQTWDLQLFELSKLSGMKIQPGKNIVRDPDKSNTAEEPTVALWDMVDVPGDIDSALQKDPVVKAMFKPHTGETMLMRQDIWAVQRWVMNSLGKLRLGKEVAILRKYVDTTHPAGCCDTGDAAIRQAQTEGKTVHSDGVRVLEDSVRMVGLDGDGKTCVGQAEEQLIQKFPCEKKMADDVFTSARALALNASTLIEQTDNGGDEWGENDTIKQVIKTGRKVEGDAAEIATPDASWNDGVYRKSAVFSSVTSDCITDIAGRTNTIVTIKELKGPRSLPMITNLRKRTALILAEMKTLIGPGGLYIDKTGIDCKVKDEGKIDSQIKYEIRGIELYGNMTPAPGIKPVAFTGKGGSGKSTIIRVKGSVVPSFVPINKFGKGRGERRTEKNADALLIPFSNAKKLEGETKVLIPDFLFKITTRNVVTVGRIVVGGVLDNSDVFEGFDSIKLVQAAMVEGQKQVTIVGINRKEGPVYGDNLLLSADIETEEYMTYGADQALAKAAILRRSGAVLFALVFGGNPNPRIFKGTEIDDVWLKIKPRAQMASVKFDEYIKQGTIAVHGGGINNGKYLVEGEDDPCDPNDQPLP,17.9744
+MDILKKLIGLSSLLALFLLTPDLLAEIVRDIVGVSIGEMPEIYIYLLAFYLLGLMLASMTTSPPGFSFLTTRIIYCIFYAWYYILLTLIVTILLIIGKTEGNYKISKQFGVTENGVIMNMIQKAWNFLSDISTGNYLITMWPLNHFGVVPWFNKAAGIAWFIGPYFTYRLSQRPVNFIFSALSFVIKKWLSKIWGKFVRMACAFTSWVFLMGVATTLVLVIFNEMKWIKCSILNSKQWFGKLLGYMRNSLTVLCQKTSIGINMMLVSVLILILIGTIGNGDTAIYWHILFLIYSAIGIFAFVFVVQIVVCNKDRGKKTDLSPAVYAGLAELLPSLSTSYDNVNLAPQDYLTALNVIDSLLIKLVLEIIVAGFLSPLLYDFRLSSDTKLIFCFISLILFGYVFLGFEKDKAESEIGHSRVPSIPRNIHNHTAATVVRLREVLYELFTSQDEAHLGAHEKQNVSKVLLFALFFLFVSTYLTISTPVSNVNCTSYRLDTPFSKRKRQLSLALFSIGCCLDGFSTMQHMVCGEEFQLDSFFKFFVRFGKVIGKRVAMFFFWTLAKALASYSDAIIAPGYSEKMAHFPPDQFNGRAVDFIDVDEDLFANGFSEGKTRVAGPGEIVIFYQIGGNKFEAMFTVSEPIKLLYYDKGIIQAEYKSEFGCELITVGFTTPTVYDYLSPVPAYYFSLVKDPTGTFFDLISVPQGSIGYVNAKFQAYGEFWFGRIVHTGQNRNYITSLPLLIHLKAGNILFWLICVVDLTTTSILGKGNKRAVEVYGLSLLSQCDSFHTEVKIIEEVKRFFSLKQRKYLSSIFYTSYMNIFLALQYKAFAMPINAGVFVTDLDEQAGILIQAKKTRRRIPPRLIFVRDRVSDPNIKIENSSPLFNVYLLSCGTDYTSKKIISIDNRIWALLDGIHKEELSYEFNYE,17.5258
+FRFCYTWVVLILVPSIFIRSFLRWESRFYFLKELERKMSGGDDLVQRSEQVETSCPVSSRCNQISEKILNWIKCEHKRVLVGGDVEQIIFPYTSSPTQSAEFQKMHQFQFLDDSGTQEANYVYQRIDETGYFRFADAAEEFTGALDVEGMCENWNVFLCMNVSTQISLILNQAKAYMFTQVLLQDGTPLVQFLDPDDQRLLVNCEDNEASNEMQDANRYQQILDVDYLLLEIQNQYYPAYFLVNLNHADCFKGTPLFTPKILEGVQDVVTCRRLVWLKFALNRYDPYDSVGSLCNTPRYMRLTRRLMENWDLSGVFWTSLTFLLGRCW,19.2309
+TPGGFIDQNREISHATRNADVNYSLLLLGHYTALAGMHAMYLGPDNVVILTEGGDFATLPYTAAPTFTAQFWQMKILAAGFSVVIAFGHFVFGVSRFWHGMLDVTMGHSSALSTLFAGHIGHLICHAGGSLIFFNFDSEPVVGTTVTIVLPLWFTHHNLVLIPWHEWTLNHSQLVQVLFKPNMSFIFAFGGHFRGMHWGIGGVNEFADGHNTGLQYHHSFFILLSLLALAVHALQISRGIIWPARNWNRAKDFWNSDTVPWISYVVYTDAFWALLAYTLGAYWAFASSGNLWTSIHQRYLQSEQVTVTTHATAMFGNMAWGGFITPVHSWIFNQGKLWSLSQISHVSRGWRFVLNSFHHGLWFIALVGIATWFYWRLHFRWGDTSLAVEAGFWNIHWVTSNAAPLMFFILAYILVATETSNKY,18.3108
+RVFFVQNLAMMLFLLLILIPLFGDKYDVLVSCTDEAFVELNYLIVLAKQWEIGCSERVVPMAAILAFLINFGLICITLVVGGNVIQWYSPKLHEKNHFIWSCLETIMVITVVLIFLQVLVCLGFTNLITALCWLPGWKLVAPWQSCALRQRITLALLLLKVPAFLGILLHVFSKQGWLIISGVQQLSYILSSLTMIAVDLWGGSIAIQDCRGKHSLIVLKVRVLMLYAPLLSSYVYWFEYAVGTCSRYFLEIVDFLVLAFMIVVLLILYGREYVERLDNIYSLVDGANVAESLTHTYILILIAYPRTNPNRIAVHIKLISFYVWMIVIMVFARKALRRLIPCSHPFGPKPTVSDKINAVQSGSTKAEWESIEYFVLFVFVILVLLLVGLGCIKLYDTFWNPQEDLDTDIKTDPFNNSLMTIIGVPYLVTVVIRKTMSLLPLTTMFKIVGVLIAGRLNLTIGWAMAYVTAFWLVTIECIRFFHPQPGVSAKPECLRFALTTATVLVASATIVFSADINRNHKGEISKAAGFAWLFAYVLLMFLAVAFVIWKIAERGVLGQSATVSRSESEEAVVYQTAKDARADIFPLVDDIRVTEPKDSAGRVINLGMPLTPSGQSKNLLPAPGANPVPTRPCVFNACWTVPDTVLVFCTVAAQMTLTFRELVLERVKDAIHTGRCVHFWPDLAICADVFDKTGLDNISGVRMNIAHGESELSAIEFNIKFQ,17.5525
+RWSLATLFMVISLLPVAFVNFFSFKEGFHFFGFVFAIITLGLSGIAWITKLRDPVDKMFFFRVRMLGWLRPTLVYFIMYGILGIISRLTALVRFKAMPIFMQLGHFFPVVNGILIFVANRPMKAVRLQARILNRIARGRWTGASYPEDKPGETMTDEEFICLTQSGMNLGDSFQIIENGTLIPNWLTSSPLKVEVLLYTYFLLGLFGLCVSLRLAGCGCLPEVIRRMWCWIWFALFFFSSFWQVFRQLSALRIALGRARWKKFDFGPVSFGLVVLFLVAVFLSQVLLVAILDIDEMRQKFAEVQTALTIPRNLPELKDSIKSTLLPFQGELQWYSQWTSLIVYLTHLIMTGMGKAMELSWQLFNLIWAFLCSNNGFLCFFQEYFLKLFLWGSAASILLFLPSILNLVQRLVPFTILIFFCVPPLIAVHSLYNRGLGVFENDVGTLKAKAVQTSAAPDWQITETNGPDEDYESHIMAIVTFVNLLCLHIIALLMTGTNSAQPLLFELKFDAKVFMAVFNGLIRTMSVLYSRGCETYALLNLLASILLKLALWFWAEIEEDEFASNISLGTLLREATRHIPVLITPVAMVNGAAGAGLTLLWPTRGHVYLRGAGNKRPAGRTSLGYMAGPSGEQFYIRGAFWMASIEISAGTGDVEINGSHIAFFALKGKMIKLTLDGKPASKIKDLVTRYADVAVDDKIDFEWDALAERSIWDKKQLKKGMQLNGSWPKTNVMPLITGQDMSEKYLYARVDLQNFNYNVKGASNKEGIHIIDPFGSLLPLVVFSSFGIIGLGFLYKYNVWQDTSTLQEHFQKRKTTISSSKPVKFTMDEPKLLGPMVFLTFQIVVMFLLGFHKFPWLYFAYIYKN,17.5249
+VYPAGAALAKAAQKALENAIQEHYEVAMREELEANPERRLIAILKGLARVRSAYAEIDIMRDQAQNAIEESIEYANDMYKEGSYAIVTPTVRIHHSIAPVEMQQAIELMADLAALGYGDAGPVVAQVIQLPGLKFRGQTHPGASPYKIDVDVAIAAINLAVERLLDALQEVYQQPPKTVKVHRVSASHDVPLVFQVQVFVQFKINGAREGDFIYPGRDVSPQNRKVEKFDERRKSTRIIPIYRLRLQNPGAAFALKDHEAITLGFAQHYLLGNWPIEVGETPRTTQDGLPSMAEKAADSTNYLLAAANFMHGKPDLEVQMILKANLCSTEKKVTKLDRLALAMNYAVYLVMLAQDLSLFLKVPQNVKVHDGMGGDYQIMMSTLPSSVTEAEIEKGGDMHVQLKALLPVSEPFDAEDVIFGQTARVDEYLVDAKMAKLLRPPTNCGTNYKENSENAQYFPLGRNINLVPCTMEDGSLGLTGLEGFSLSQAGNQRMWAVNLIDRQGKVALLAEFVLNELISLSDNAEQIADSHTLKVVGVRGNVGNLWTTGTMSDKIEYTPVLMSGETKLHASLGHLIASTPDLTATMQEKKLTLLLSPPAYDETPPKIEKLVWPYEKNAEVTGPWRNITKAIGTSISDLLSLSNKMAKVEQERDAKMETSELQKYDNPEIRISRKLVSALVIIISLLIFALRHRFALATWRGCVVAIGTPSTPLKRLSGIVRQSADAGITTAGSKCSRIRIIIKRGIFMTTLGISSTIITLVFAYQ,17.9413
+YMKRGMVHRTLTLLLIFMVLALGFAIDIRGWAMFLPEATLMVLSMLGFFRQGASDPNYDAVMPGVVCEIHMRTSMLFFSWSIALAYLAMLVNSAGQITQSPKIVDSFTKIVSAGKGLLALIINGITVAKPTDGDELFSQFSLTLTLTNIGSMSIVPQIRLQIYRWLMKPPHGFLGIFPVLSAGTSLAIALFNFWRNSLTEQYFKFLSDMTNSINAQVASMVAHRSMAFNWIGTIWQYCMITVFLLGFVYFYFLSETNGIQLRLDIDHSCGFSNVPIVFAHEFLTIACKAAAEILKSDSDNKVKVHVMSKAENIAGSGQLWLEVFESGSLPGANASIIQVIPNRKREADIVGPGTVDGLDGATLLLSPNNIFVPPGDLVAARGNKAAAGATLGANGTLTVDARKYGASKINVGYDTEAVGLAVCTLVIGTGDSVLPTAKKQMDNVVAFLIKEEDAWALQQPLKRSAERGYFALTMAMNPDTVAFATETGLRVCDLMNTLGDLFSIGPAGLDAEAVGAQGLSTTIDMNKHVFFGLEAKFSSSNINQPSSWLGALEAGLGNWLSLRNALRGDGPQPQGP,17.4011
+AAYAQKNAKIKRKLEETVLCGGCDEGEYRRESSFGAISASDGFTPDWEHNLDGQPGLYVTKLIYKYIQHPQYLYEILAVALLGVIGAKTSLFEGLSHPKRRTESLAITFNSAHVSACLTVLTDYTRQLTYTLSACLVTLVSTLYAVNLIVRDKKIAADIQFFVEASDYLKMGLEVTRNENVTPVNDDDFFSHILWLIDHTKPTMIEGHFREYKLVNKFFILEEHGLVGKRGSMMDPINTFIKCEKLLQLIDTKYGGSVKKLKSSKVAFYNAVSEECAPVKITLPKTSDILAHRYVSVRDIPARGVPYTHSFSSNVVSAITDGRVMDKAGDFDEDLAIKIMGLKLDGFTVMVYRLDGFRMGETSVSKIATLEALIKDDIVTHELITKSSFTRDYRSMERHDFVLGSNFPYCSPAHEDTIEFKQKRQSYVGRAVADAKVEELELPATGDRGEVKDQVAKNMKVLTNQAMKVHVGTMLAPDGDIYSITKENVLPACYVDVKGYLTRRNILGKLKKFMDVFEDFAKVINILDDTGSGNHRFNRYWTRRDSRLGKPLLLTHEDDLETNVADNRRIRTNKQRERCLVRVLNLECEKCHLPEMVVLGIFIGSSAILFTLFTLMSINGVNVLLDQVPPSGFGASIEGAMREAKVLVRLGEFVANKANMFSQERGGDVPAIVPMTEEQRSDLNKPCKEERKISKCFTRMHGSWGGVKRMDPPFTRGGYLMMRQTRMGIWISFDKRKFGKTQKFKYLDCGMKDPNVWKRNINVGCHLVNTYADTNFNCCTQTIQAVVESHWTEPLFVARTFQPVSICLIGMLQFSYGPVMAGLKTPKPHPGTLRVVNVSTVNLMLFVLFNYLRPAAYNGFYGKYTKPFTLGVSQKPRAWSHKIITPPGPKQDLFISFFSHLVVLIFMVVMWIYFAGTVTPFDFQYYRQVSLDVV,18.5249
+MKYNNYALLSTVTILGLVFTIFNWANDWNLHLNFGLTTYLFVGGTFLILTTFGVGQDDPSYLKGFTINLAGKMIIGTHLPPLVPTPFSPFIDKISKHLAGAHVTISAVTVDNIIGTLLKLLESGDNREGNWRAHRLAFHAIGATIFLWEIVISIYIGFNDGFNVDNGEKVSKTAELITTPSGTLDHTGSAESWSFDDDSSPLKWYNGFLISKEIKYKHRPLFSLTTQLSILMYNRIFIVLQLVIVHSERMKLSLFDLFFQNFFFFSTIPLDLEGLSGEYRSIGGKSDIRTLIVSCFGSLLHG,17.1279
+KNEKTNSSSKKVQVADEIYAPGKPVAYVNTGTTQETIASDAILWLASEFSAIIEIKVVLFGPVTNDVYSCNIANYSPIPQGLEVVHKKYTNKNNLWLFTTGYDLNITFLNTDMLNLESSFLIIEGAISTSRMTSDKEITNFEVPGNAVVLCTYNAPSITSKGAKAHEASGGLAANLPREEQLQAILRSHEQYVSRKMKADCFPTTKAVNDGRILLFYLSAKNLVDSLPMERGDFNLIYQKMEVKIYLDDLLKTREEIQAARAFMTEFIVRQNGDIKLLGLSEISDTSDGRAEVLDLPLESGNSLSSEVDAVLVVGQLRAMIHGTGTFTGVFIPHDLISSGIDPESDREGIGNFSRFDRESLVLFGIGVYIDGVNEIGWFKKTPIAIGIGNASYRRSNCLQISFYCDVDANTHEDTGTSKGKTLIMATNEYSIAGAICEACGYDVEGDDKTDRQIVVNQPSAVTVAGMPGLVAKTHNGFRNFEKNFEYLNFPVSKVLAEEGGLDYFWTIPPGNYQNNVPWNPVRAQMTSWGVSTTATLVFGVTYSRTLLVSLRVNATLTTNSLFAFFASKLSHINTFRTGGIISGGLCSVLILNFVVAIYGVSLRAFGGALLTYAMVMIVVLFCREVWKVMYYADIYGKQDLIIFELLNFFVNFGFILTIPLLSTASPPGSIDIKLPGILRTLSLYNDNQQRRTFIGKLLWDPESKVYNLKSGEAKLLGANASGLMAGGSEGAVHEVETDTSNLVFRSDVSSP,17.7156
+EGEVNRIVLDLSGTSDGSVIIEANKVTRDNVSDALLKGKNFNAPAKTSSYPAYVAASLERQDPKKTFISFFKHAHNNAHGGQGRIINLAFAHATQTKRFNVFFEAYKKHGLKFDQNTFKFHVPEDMSRKGTIAFKGNDGEITLVDVFTSSFRQQISQITIRQGLWDWKSTRKNELGYFNNTIQFQGSKTTGSADLIFALSLLGAIRTIREYYPFKEQYVLLHRTWTNLQKKNKASWEWASARDKGQLNTGTKQTFATSLITELPIKSFELMTSARSLPEMEVNQQYEHYRIREYYKCRGAGEDTLMKGITGSGADATKVLMISYMLNEGLVLILDYSQQNIKTGNTIAILKEQGLAIKTSPSYSIQRLTKLHIYAMENLEIFPLHREQVNVMNAVLLGELGVAADEVSKANFNNMPLPSRQATVLSINLQDKDKNRKVLLRALGDQNSFIPPFDKSDVQNTVNLMESITKNQAITFDLRQGNGRSNQLIDVDI,18.2889
+NTTRNPTENMPTPRSWLTEGRPYIAYACAKCKSETDKANKGLLFVTKDKIIIKSVPGIADQIAREVKEFFNVQTPAEGWDLVVGDVTADASAGVRGVLGGIVFTQKGSVLQALAVAVTSIGTMILFLNLFSWGGGWVTMFGAAENIITSLAMIAKLVLENKVLLLNIGMSGAGICLMMTTDVSPSVLTANFLAYAMIDTIAFAGDAITYPFTIDIGDAFFKFYGGAEIESVYNKQSKPWPSWVAEQLSFASASNTGAGVTWTFSLTSINKSYTLQFLQAVGLLLSAQSRPPDLLGSEAALTFNVTYVPLGFEVNAARIKTMLSPTKNVNQIGNLKPFLKHLTESLGNLKTLLKQRTQITEDDVDVRKLATSIEVPEKNLLNNELNKIRYANFVSKRLAVALNDEIPDLYKVNDLKSSHIFLKSNANLGNGIERLNTGIDIVSNQEPQMLTMLFLKGRLIKNNGTTAPLTLWLKYISNLTTLTNIGISVEESANRFRNLQNSFFKNNNILEINIQVIANPTDAEKELNLVGY,17.3426
+GAQNTDVILGALSNFILGAFGLYYWFQWGNVILHQAVIMSFIHLVLSPDWTIWFYPYFVSEGCLYRVVLAIVQRTAMTLHISPEVSKYGIRAALSSPQEMYSLSRGDLRWFFKDLAIQKVRWRRMPPAVMILVLFIAYQLLQTKTITPTQLLLIQGLLFRVYGNLMITITILGTVMGVSPFTVIYNGWGKPKGITYCEFPSAFLFLLEDYGSGEEMTSIALPASLFVEYTEKASVIRAGYILSQVDEFSIKNMITRERNLPKSELLYVAADSGVNHTLNICQFPVSDTYLIKYSFIPYKLYIEDGKKVEMPPNKVWDAIVIGHYSQDDYWQLAAFCNQEWDFANFEKMLARPQRLVDTCGMALAATYWALLVQVLGAPILDNCLWINTFAILFAAGILWQIPPLRQDMRIDLSARFKHSVIVVAAYPYVLRLTWSGQSQQKFDLFIYFFLAIFTLSFNSVHYTADPAREQFEWRDSTGKDIPCVFLFGLTVTYWYGALHTGHDPESNTGLSTAKTSFDWKSQFQPFDNQYTRQATELLGIIPCATLHRKCRETWTRQRVFNVMVDMQQGSARFIFLIQDTAFNRNFKGGLIQDRQDLRKMLAISPGEALRAVIHRREHAAIEKQLNDVRADELVVAAQTAPGERVQELLRGSGVSYSLTNFVTFKKNISDDERRVPAPELVFQIVIVCCWDSRIVKALLAIITITSLAVGDLSGVFILFRS,18.2048
+GELPALAGNRCGEAKLFDILARPDLPRRWYIHLGSVFTLMLVLTFLGAFIGTGCWVDGGGFGKFIDRGLSQAPTFGPQVLTHLYPEAWAHFFGIADPAGGYWLYHIILFSGAHGVFIFAGGALARTLRLGRLLGMARALGMRPKHCAVGAVGVILFLTAFYYLPDGNPTFTPDQGYESGSTGTIMVIDNGAVGLLFHPLFGAGLTGTFHTLTLAHEGTASGEGLSNLSEGGTESETYAAARLNALFRLVANQGRAWRALHIYTLPFLSLGVCAALGLTVAHAWTAFDYNNFVAAARADSFKFGANNWVLAANDIRAGAGKFVHAGDEVLPGELIR,16.0959
+ADFFVRRQSTKKLYGLPLDGSVNDSVACIWGFAVFWNGLVFPWVFAFVGLIGWRLQIRFVPGSVIGLFKFELILSLIPDALAHFGVEDIYANPEYVFNFPRGVLTFASTHGIRTLRALRFAYPFVALFGRKAAGLFRRMGVVCLMAMVIGVGFAVAAFFFGELMPTMRWTFGEGGIIQTPVFAAGFRSSDVPATALEAAHFLVFFLLGLIFMAIHTGAAIFYAGESAARKNEDSQTFSWSSASSARLTRQRDREILVRRNGTSGESPGLA,16.437
+MSYLYLVFFMILILFHLNLLTYNIVKKKPPFNGKYKKWEFKRAFDRYPVGYIYYGHGQWKDERNKTEKHPRDQ,29.4763
+KVAAIGVPFFGLLIALLLNITMVFLSQTTLSKYWFAWHIFAIILILLGLLVNVLVNQGSSGSTTSNFDSGMLAMISVGKALGWNIMARYTPWQTGTLNSISWFNIGGAVTVAVMGKMAGIELIERENSRTPEGFSSPWPVGQTPAWMGAGPIGGVIAIVGISVSAVAVSALANISVVDVSNISLLLEIPVNSIIMGEGVGFYYLIMVLIMGMITLAYSGGFFSAKFGGYSERLGADLAGARTPLNVYGENIPKVMRATASVPALFRRPVANLALSLWILASLGVMVTYFESVAIFNRTIENIGKVAITNGQSVDVMGFTDVYPLDVDESNFIAWRTAIPPGVLVGVTPPIFGRIELVAVNAGLLKLERKGVAQVIDTGPESFELEAKMLAPSMTGSFSTQAAIGGSAFAAMFQSSTGANSVFVSFSKGSVAFSIMAGVFIGLVVALMLAGLNWNPGTVMKKLMMSMTVVSAGVSSIFAMVKPLALTTSSFLLVESVVIFSNSIGASEFIGFAGGAAFMVNKQFVRALASGTGALVIGGPVFAIGYIAAGLGSVTAAADVGRAAFIMAGIAGVLTGVSMLTGSLVGSAKFPDRSEGKMKVLRNWWPGYSIARLAGRFETSNLLMFFTYVADQLGLLSKDLVRNAHNFAN,16.5548
+NWYNIRAHNYVAGTTMVDAATKPALATSIATQLLGTSDYDTISKLEHNAKEGGKINLIMTNQFPASGKMVIQQGYFGRGSAVPYTNRLPLIQLLSLVDSAATADKEQVLSVGWAIDAIVERRASKMVLYNASKSFLLGKISNIMGSMLVNIQISAAGQYTILTSYDSILTSKFLSYNRPVVDQGAGMINMATGTTVGANGQLLLRKVKEYITKVQGIDASLLAFAQRGLGSVTQASISARRPTRNRMEENAQKGAPGEFSKVTDAGGGHLPGSKMVFKRILIPVFMRYAIMDVRVKMAKTTYCPQTQTPFDKWYYTLNFTLRGTGYTTVVANPDKTGKDVMRTTMHRADCTGFEVAGSVDLGLQDIQVLEMGQFKNFDVYLFLGQGEGSDKYAVAKLTNAPPIAILNGFSSTMTLKAIWYTWRWPTMTRFSLAVLYFAAGHIMTRKFQNTAFMRDGQARQV,18.0029
+DFDMPDGGVVTPLKAGETVGNLSAKGTLFNPPDDLHMRGDHNETLKYHSVTAVVIAGLQHEEIIGTAQDESCGYSAEQNTHCVAIHAAHKGDHDSSIALETEKVAVLCGDTEEGGYIWKERRHLSDSLLARIKAMFDVRFYDSHYGDKPGMSWPALRPWMKRGDLRGAWPVFLGAGGFAFNLGSMLGDGYTWNIYAILPALNGLQRLLFALGRPIRAVKYVKDTFDGTATTSFLLFYPAPSVFFLIAFFFGAISALAAGYMFLLEGRASLPQAITASIVAVSVCWQYNALFVGLMLVGEFCPRFAGTPAGVMAILGQMHDVLPHLLMVNEAVLAFIKTILYLLSGSGEPPLEASQMEYSAIVGGLVRITPAKDLDDPADYAVTGYAMITLVGFAIVLAMQVHLDGMCGDFSGVRFANPLHVGVKVIFNVDPDILCGPDTVTVGTLLFWAGGRFVFFRAASRILLPVFLSPVYKRWGSRVSVVATAFLTCTIIVGVRIRYQNDEVYANGAIYSRSDCAPGMFEEDKRFRNLLPTLEYLNINCYFYKLKGHNQINVHTFNWASMVFALYKKKEFIKQALLGWLNGDKIDLERQKEKSPNSENHDSDDWRGDVTVSGFTRPNCGHQRTTTLLQKVRFRTRCMMSRLLHVPFRRVAVHFFSFVFIMRLFSK,17.9999
+GRFRTYVKFYLRFGACHLPVTVFVFVNVAALVPFILIARLKFTSDPVHVTVEMFVEGMTFLTGSASIMLFGILMAFTDRRSELMSWWFESEGATSAGLYNEIGFWLFITIEFGTGLIGFGLRTVEIARALGFKPVINFMYFAPLMGLVSVLASIRLGMALSLALDMSPVVIVLTGLSGRDDGTNFAWLYGGIGGSGTYGTGLGDSPGGSSFLAVMFARGVAKLGSKVPEIAWAIIYALLPAVLGLGVNALPKYYLGELRVTGIRGIPFGDPAIVTRSLTKLLRQEAPVDLLVEPLLIRHAILVRSVRTMKIGELVQIRVDVPLESFEDSKIRSVDDPLLDGDDVISTTGQ,16.3933
+GVSKWFDPSKVNEAYSLSLRGDKYETTKANKTELFGEISLRVKEYANLSSIYYSSTSGYKDGFKWSDNSSKNKKVKLFNHFNAGDYQAMWEASRYIHLNQAKDCTLSYSAWNGTDAVSVTQAAGDSSLTLYRTINSTNDTTYFLLGSMNGGFSHQEQTDCSTSIPNCSAQFPAANVPTQRATYCVVCSLHNDHCKSTDVSEGCAGKNLLKESCQASFTNYKN,20.0769
+YLLWMHDKSAYMQKSRTPSVQWGYGVAAVEKLAQWWASAKGRGGWFVDPPSPKVQAIPNGCLRNIASGFWKPPVNYSHETSKWKFIYVTLAFENLYSAFWRFFPGFMGFLSPEWNRKANKWNVVGKYDYLAAFVLKFGASYTDQTHIITWARGVRDRISNISLTVYVGANKLGNVLLSLGGGLSFLRGEFQPYNHYRAKFQAVALYDWRMSMTYSAKYLQVLSGQSGLKETVMTSGFHFFRLTAPASVFRTSQRTEVYTLFLGGLGEAQKDKEVYYITLPTLGITYYSATLTGSFDFSFHVGLKEDWRSIRRGHITLHFGAGSHDGKLTLRNVVDITRGIPLKYVDFRGLEFKWRDKAFYIHAKPDPQAFWVGIAPSDGVKSKIGPLPTITRLTPQLLVAIDINYPMFPKDGVDGYGAVEGESRSYYVHVFTAFDMQSLFNGQVHANYQKNKPKKDVIVTAATTPSSEELIKQLTQKCGKRATFMSIDMQDK,18.4373
+MNPLPYKVRFLEWTNDSAPDTCSEAATAEPALRCSNIVGVKNPREFDTLWEKRKTRLESGTLTTKLESPSRMAILKRSIFRIFINFVVALGALVLLVISVSLNVRNNLLDPAYRIGVSQNKIARIGIDLFNGPKLQVAEFKICLGQTVFHLNVLHTILGLLVFYFTLGGADEDSARYDHDQYLPFSFVTNYTFHFEVAHYAMEQFGVGALANLLFLILVAHTIFVVSEEIRRGMANRVNLKKTSKLNPSGPARIIEEFQYCAYFVNQVLKIGKWAEPAAAQFIGRHDMIARELGQKLFDDNPSQSEVNEGVTAARVKVINGCSKEPCGKPPVMAQDLASKILDQFGTYSDTPIIGRINTIMLNGNTENGQTVIDGWLHHLQQRLEVHHIPLAESYDNFIFGLDNTETTLFHPFWTDMEEGEYGNPNYITSGERLINYRRALHNTWGSVFLPLYVFFWNWSILRPPPDAETLLKYQISMPSSIRATAVIHYHIHWLTDEEKHYVQGKITQCQGATIICESTATEDLIEFVTLDPAWSHLTGGRN,18.4536
+MRRANLTRADSIADGEVDSLVRASPSLPRTEDDAVYLDGFERRAPDFEAIAQLSKMRYAGMSGLMDELKKLHDATDLNELISMGEMALVESENRTNAIVRQGLSEVLAAEDLSICDIQIAGESGSVGFGRGLRNLTNYVIDVEVRPNGHLIIQAQCFHTEDKSYEKADSKPLDSVQYDDRKVGYQGDSVNAGIPEVAAAGAGRKVLYAEIAVGGDRGDTGWKLAPIGSVLGGGDGAGIRGWATAAAQIYNWTRLAEGIASIDRGLAINGGARLDGTQYALGVGDANQASPVLFTGGLTGAGPAHVRQFERLVPDHPLSKTLVVLSSINGTVLADNSAVGHVVARGNTGLEILTADTAKVANGYTLPVRGEFDVSSAGNITAVTAIAGPGEDISRQAP,17.1274
+GSTKDQKQTFTSFVGWIIFCSVATLSSFVYQQVLLKGLSQVLDYLAVTGSFGGIGSILCFFISTIGSGSGTVRTNNLYQHAASIFWTIIGFFGIAEAAGLVASLVFYFFQ,15.303
+SGLPAFLAGIYPVIGGSLAVSIAKIGPTVPILQAGQAACHSKLLPSNEKPVTIPVILSLAYGVLGWTLGGLGEDLLGELGQVIGIGGPKL,14.4576
+RTSQIFEAFLLRTKALKWCWIVLHLVTLLLLTSLACAYYQVESAHSQQPVLDCAYHYKRLGDGWWVGYSQGVIGFGVTAFILLISHQEASGVQDETGKFARYWKLNCTIFLTFLVTWIGLHFMIEGIDTFIGYILMVAVASALLGQVLISINEVAKTTLLGNNLDGITLSYGASPEPVSKNLEGDPAVYAQIANSGISIRLWWIIWALFAALGILLFVMLTDRHPTPQPFVEAGYLEKGIMTVLLLALSNYPILPAVFLIVLTSADIRTHRNKVVYSCNDSKFISKLSAYFEQTNKEVTVMMETAEPIVHVGNYSSPVGAIITISAIIVSTLGSLGKRKSAFPVTLTFVVVLITVIAIANNVISPSDQPVGDNSFFLFEITIALGVDFSSFILAICSFLKLELNTIFGSFPKCCYFLLSFVIMLFSSETFIAEPLFSQILLALISVITLPETTSYFGQKAVSFIKFPCIKDGFSILPTLLAVLELFGIVRNLRLLRLLRSFRAFRIVSEAKVFCITKTVLAHFYGPLRHRLLMHTVKGRKEKLMQALMCLGILAFLVSAIVEAIVLLFASYYLSTCYLLPAFSFSTVTLSLLHVYLSYIHVNTILVALVVSIFVIGILMSLILRIHKNMKAQANN,16.9812
+VGSETIGAPIENLPDPLQAPAITAKIPTGATVQYLAQEPGIVGVWLQPRMVAFKVNRAIGSISFLIFFFLTTFAWLYITPGQINVVGTCVGVSVGGVLIGWGILIPGDPAKASFKADKYRWVESLALKFGETAARACYGYLFLSVAAGLEYLNLFIF,16.1419
+MNAMHLVRLNSAGRGSSVAILNDNLATGAAGVSSHMSEDDRIDVIVDFSRGGGGMQQEALAQYLTARLSSDGFLLADINKPNVNIQSVATSSQFEVQPRIQSNMDVLVINWLIQGKDSDFSTLIIQRGKTPYINSAHREKILLSLNSINVADKDIELDFDGYQTGPTQQLPPNVFAASIGTSLAIFFAKGEIPLRYMINSETNGIKLLQYISQSSPADMEREVVLVNHEKEIQQSLNTEKLADSELFLEGWSEKIDNSVYVANLFEDCFHRAVVGCVATARLDDMMGTVEFAAWLNVDSQGKLLISEIYTSFTPELVAGQAVVGGKFSTVDISTGEYEIFEKRAAFGINTQTASALIYLPMPRALAPRVEFWQLIEKLMKASNQSVMISAGVAGTFSGGRGLLYVNGLNAQLVGMLDALLKLQKIFAANLGANPNLSNVLIIGDTDSVLALSQGIKLPNGMSLELKEVNKLNNTFLDELSEIIGDFSGSSEVRSKIWTSTQEVKLGDLTEPLFVGVSSDIVALVANGNIELIIANAGVSPRANLDTAQVFQRGKQVIKSRTGPSLNAKGLYLVLSDQESIRSCQLTGAQNLLAMNIQINLKVVVRDVLSAAAMAFLAKECAIVDIGGCEVSAPAYPEVVTLRYDTQTSRSFGQRIIQKQTLGNAAVNCSVSDAGQSAPGSSGHAKGNNTAYISVIAARVGGGIGDLAIVLAGLIAGATAATAPNLAYKWKGNIAPQAKDVLSSVKNGDRSLNTRDLSVEPVKNELAGTTTLNWHTTFAMNSDSGWRNVHPYPSNGNFP,17.5854
+AKAPGLVGLGIGSVSGLVVGLALSFLLGCVCTDHRWAKYDGAGLAILEGMALNDALLWVYPLQWTLIGGVSLDSSSVSLVLVIVACTAALAGVGRVLRAILRFFAPRTRSQRLLLALVLSEVAVQLVVFFAQPLATALPLITAFTDHTLQVCYGGYTTLSPMDTLGQWVTYVKANSTGGTSLRDPYRALSILLVSFGLVTVAVGVTLKRFTASAGDCQ,15.0093
+ALAQCVLLALASGVSAVLAIIPRKETYIRAKIVSIKKAKYGLSMYERGGRLKGLGIPPWSKAPRSNHHLGVYADEIGILGTIFGYTVPMGALVIAILITFAHLMPSYIKKYVYLTQVEIENYSPVPHQVPAE,16.8979
+KECARRIKGCLNFTGSASWLSFVNLFVKQIYTGYVFAHASLMTLLVWQAAMHHIVNMNLCDEYHWTFTTATSGPLGYRNFTTLAWIDSMANFVALHRHFLVYGGLYHVASASLFTAFVAHFIRRRSPPTSFWAYLNFEQKKFLSAYSHGHHLILGSFLAFLTHLDYFFDKFSVHTNAFSQSWVFRGELTPELAVNLGLMFHVKHFSLFHFSNSVLILALHFSHSGAFIDEMRSLTALESAYGTMRWVREGMGWHRGVVERWYHGSFQVKHTEEGSMQFAKNFLLYLPELPRAECYAAFYLRTDFKGNLALRRHAEYRRKFYMMEGKTLFWRATQKGLECQKTWGAGFARTAALTSSTHAVAHVANVTTGFVLGFFFVWRQVHK,18.4019
+FIATDRKWIPLWILNMMYTLSGMGGVVAFSAFLAITRMGYDIKWTGALFVAGSVIEYVEKLFPQAGPAGTLVVLLIPAAATGHGMLPMICVVCCMRIGFIGWGAKILILPLLGNGKLLAIYGIRSPWGVAFTVPAVLILVAAGLVFEHTWKLVVHAYDIGFLLTALAVALLALSKLLWYKEPILYALLAFSVTALVGFIASAAGSFFVGRGCTTCQHPPSTIFSDNGRGKSVWTFFLAIGVTIQLLALFAFLPKVGTHQTVKDLFNIIGTGDITIMLEQAAKAKRRGVYVLNLFNDKCPKSPAVLDRTVSYLPPNVSCAVKATKDTNMPLVTLTDDMHFHLEDYGHRALQEFPTLPFNHTKCYLSQADLYLTGTDMSGIILVSLNNWMGEIGHLAAHTLECPDERAVFSLPIGDDTFKYLLYHEQTLKGIRLFSHLLSQSSRQVTEGAGGRDILMTHQTALITLRSLIAAEVFVMTNGTLKLIPIGRTDVLWEYRATAYHDCTMVGIPSTNHLTWDGQVVESEPLWPLSGYMSLKTGSVILVMIDTVGRTTYMLQNCIIYLGLLTVRLPASVATMEDRDCVLMGYLAFLVKTLLTEKGYCFLRRACELIAIGVFAAWFMSIKYIAVGAFTGGILPWVLSYAVLGMMFIGILYCLIMFRMGQMVERGIVVYGRSDDVSMQNRLPDVADPMPATSLVLSHEMFSGCLPNDVHFEIREPVGVPMIRFFDWYGERVLPCQQPFKEVSKLIALVLQQLAHMHEKNLDPPIWNVLRIHVTPARPFRGLGAMGVNVIISYMILILVKFLGITISEKWL,17.7597
+LFKSSKINSRNPISMLNIKMNLGSRYQVLAQIQLSPNKINSDDDTISFCINTENLLSWFLPGDHFNFADLRVMWALLIVTIICGGILFATLSMLYGIAPTRSTTKMIKINDQPAFKLGLIRTHVTFSSAGILLVGVSYNLSEELVKVPYIRGGNLYFQFSTPFALEFTSICFHNSYEPLYNWLAGYDPYTGTEVFFNFGPFLAAWGAGVAGTIALVAHACVAVELFKQLKFKIKISKICSTRIILPVALTGALIAWIVPLISSPDILKTTGKNIHDGDTLIMIPVLLKRIFAQMGKPESHSIEHALAHNHAAPSEAQFRLAIDDSYYNQAISTCTSRELKPLLNRNVVRLLCADGKKTIRDPKRILESYCEAINRVFGGTFKDFLFGVVENSKLTKFFKYFLGVLDIADLSNYSNGALTTEADQFLVEFLDIYPEYHKFSQNKTYIRK,17.6359
+FWNWRRRFLGFLIGVVVTLFFVEATGTFVDNWSTIRAMHKMTGMTFGDWLGTIEALLTFGFLIAHLTGGGTPFGWVDDVFVVVTIALFARQRIFRLALVGLRGFRLERAGSTLKAVGALRPLSSTRKLAAWLMGWLSMLAFFGLVTGVLVYVDVRGNWFETAPYTFETVTVVYNFYQEHGYGDDALRYGLALSVLAVSPFIIGILGISFNWLVVPLSGWDYD,15.5532
+TLLICYGASASNYSDSTRANAYLNMPITLSDVVVGLIYAISLGSVFQVDAILLAVILGNIVLGAVAFVVASAVATALERLVGRVSLIPAFETAVSGSISGDLSSYPDLYKRTAQSVIAMAIVGEVEEQVRGAENAGEGILDVLDWQEGGGEARTTLNQIGDGVLQGVNIGEEELRSLKPLEVGNLDVASDVTDYDKAVKIDIQFALRRARAGGDVVLLDARNKSSIDFGTDIIVGTAGFGPVGTAPFIELAAKAGFNVMVRGGIEDGIALTDIEVVKHARIKGVAISGGTTASIGSAARRIGRARISVSVGKARFQSLKKVCDVAALDIQETFAVEQILLLATGGKQIRSVSSAIGKPYIQGPDGSLGDLLASIENTVTVVSVKQNKAAIINDLGPSDLASIEDRTPEEFLETTEDDVAEQNDCVLMNALGLNIVFEDNVVLIMDIELGEFIPAGREVQLLNSRLEKKQSKLRIAEVLLTLSSRALPGITRGNYDIEYHDLAAFALGFQPVFIGSAAREGTTREALIAAIILLLESLILAGMAILAVGVRKLVGFQVQPFSGLFRAIFSVVIGTAIVGLGLAWAYGPLHRLGEEEEVAQTKVGWGGSFALIISIVNAVLYLTAAIFLIILLVALALFFVVETIDHIFTFEIENTVSAVDTFLAGFGCMTPQLQKFNRQLHKIPNFFHTLDEFKGLMAHQDIIADFNRSIASLFLDYINAVMIFLDGEIATKILRALDAILGGVVFSAIIIGAQASVADSIITGRDITLELAVALLLGAGLAASVLALGVGVTIAGVGGAEKTASNEGQARNCRILFYFCVGCTSVIVTGVAVAKSIEIL,16.4406
+TLTFMMEGTQAWIPWYIIMMVYHLLTQTYNMAGLLLFGLLFAGIIGLLASRPRLFDLEQRERINWTMQPLPRALTLVIYMLLPFSSLVVIFAIAEATTYSPPKQDEHPHRLTTAINVVVAPPYNFDAGVSWIPLALLGLAVALLQKVQLTPTNRYNRLFKLVQFSQININLYSGKAPITIAMDSKDTYPIDETMRTFAVLRETSKKDTVYIPVEVENCLKGTESLYPAADTSVNLYLVHGGQNHFATKATMHSIFIVPVGIVGPFLAVVHLIILGIAEAGKREEYYLLYLFGYLSVLTLKLNGTAIIDALIRDGIHCARLPGRYNVLNYVVPKVSAEMKIIQDTLIYWEPAATQWETKLFDKSRILRNSPGYKFAKLLSVHLITMAAYCTLILVLPTVLSEYGQRNSGPEKRVLFSCRLLKSRVKGKSRICFHQVPGRKMTDTAKKLTSGVKNIFRNPGYKYMESNEILLIYTINLDYKNNALYENGPAIQTAAVINNNHAGTLFLQDIDVIPNLMALSPFVFLVCGYAPEDTELVFCWVLNKCGGKEVYIAFSINRNQIEDPLSKLEVIANNIIRVIKDDYNHRKAAAKEYAEWIAVAEASIGTLPLSIAKGFFASEETPKELRMSFILRAMKQWLVVRRKHKHDCKNMDFKQRCKSVATIKRKPMEQSLCVPIEKHQPAMRKYLIITLLEQNLDRWAHEAERVTSHFLPFFNNNSETNCHICECLNEYYQDAELLLQNAKISSGCNEYGAIYYSGIPISGAVAQKMTNIFISGSSVVLLITSYGGDE,18.2811
+VAQMEFPEGTTSSCIWKQGYHNPAVVIQQLTLHRCSASSDTICTLMTSQSNSTQLMDDLLASVLKILVGLISSDYTLIDVGGFTVDVDSLSLVYRNFHTNISPCNIDTVTKTPDDTLTFEQYRDDMRAQVEQTYKAYVSADPPETRSVKASSYTHVYRPIGMPHNIIQPIMMIVEDTPQTESGTGIKMCSNQRQDVVTGNPVEAFQTLAQGDHYKLMDSSTNKSILAVSNGWNLCLGSFDSLENNPTITDKEGEKHYKFMKNADDTLNSYLYLNAVYINDPTLPVTILSSNAGCKGLLEAIYKNNIRILYDSYPAPNAEASGNNVKSIGTITVASCMGKGSMCPCGDDYQTLAVAVSLVNYEYWDLNGSKSNNNVIRTSGTFSIAILTDRGNYTANRSALLKAYINLLENYAEERKKQIKATIWLYQRDGRSSGKKEMSCNDDPSDTGYVAAEYPGAAQVLDTDDLETMPGSILPSFQNFAQVKLFKQQYKGKVPVKWMHGYVRHNLKANYFANGYYFAPSEGSIINPVLGGATE,18.9275
+TGPYKKLADWNERVPTPSITQYIASHYNYPDLVAVRRVLRVPVAVDATGKEVTVDKQDCFFKSDGVVYTTNYKSYPKTIISESYFAYAIPGDVQQKMHTIPLTSNVYKDDREFFQYKVSFQFTPPTPPDIQYPARADNDSEGVLDWTKEVPWYAKDCNGPLAKCYARVNTDEFYESSAARLHPWDFPWASRLHIPAGIYYR,19.3561
+SRKILPIVGLIIGIFSVIAMIFYVLLKDKNHATNIETTPADVETIWNMTGLLSQSIEKAYTNPTREYITHADVLEQLKKTFNFDSEILNKAMNTVTQYMSENQGDAAVKLTEDFFQTCAIETQTRNPGQFASSYGPDHKLAKDQATDETIGEDNKSPFNDPTVFGIMKALLASMTNIIKIAMETLNLFTIESNVIQLLPLVHAMNPRSIEELRLTLSYFHKNLNVTLEEDRQKLASILEILRHLLQLFFLYLFSVQDTLQNWLLNIHFNPPLETIVPTIPPNDNEIAQMLIQLNTDSSSHLITILDKASPKMHMIVLGEQILNQSLKDFTDGLHSVKDWAEPTDVLTRLGISPIDNPMSELSKLWQNVLLYIKHQFTSISNSSTLIGQLKTLAHVRYQLLEIKPALQSLASYYLNIDTTMIMSYNLYAFDELAIKENLEEEEIHSKIPEEQDYLDIIAQDDLDLYLKNLIEYNGSIDQQARNRIGFSVISFVHNKLFEILPWLFGKDQRTIKIGLVIKNIKGYIPGLLAGKMEQIRNLSTDENLQLNDKLVVFSGMKQTNGFAKLTLLNMSPLISYYFSSKAAGLTWTSDFIPVLKISQLIALLQVYFLVMKSKTITGKYMLRYTDTAVKKNHVFSFHEVAGHFEGQYSSPLNAFFNHISRNTIPGNRKIIQTYPLLFGSLLAVIILLLILKLSLYPVKLNLATLFALNIVLVTAFLVKTGKDRLKATALLLLGLAYAADLLLGFKSFGGQGESSSREHKLANLIIFPLLMIKTIFVIVSIFALYYIG,17.5915
+TPDSEYMSQTQNRYSENTCNHQYPTEWSEVIDHTSVGILVVSINRFWHQDKCHQKASFLREGAFFRAGILLGALSVLLCFSKWSVPPIPLTLLINVYVSEQWIFLGLFVGDNNEIANHYQIEVLLDFARPYKRYAYEILILFSTHVIIAVVFRNLVVYSPDSLLISISQTDRLQHGFCNLSAVLETVGILDIIVSLFLYSLAETSVALIIGLVVIGSAFAVHQAWAGMWIPGRNTSRVLREVKWFVIIFVAGRCTLWFYLFSCSQNNLIRQTSMHVFVTGLQICHLFQASAPHLVSYLVHLVVRFTQISSVLRRNNVYYFLGAPFTSSSSLIGGIAICVFPDYSGFEKLIFLSENAILMIASNLLLRDGPRRAFLVAREHQEVTLNWLSRSLIWRKEVDIVLMGLLMLLLIVATGNIFTIGEIARVVSSSITLNSVLSLIYWFIFNGEHVKPFTWVSLREMVNLQLNLSVMRSKRGATNRLQKMNAIQEIMAVDLSGGQRRAVLIARELAIAPRLVILDEHTATIDTVETSVLALLSPLLRKGTTAIVILAMHGRDLLHQLIGLIYLNMRVLDYLRHKKWNDMKLFKAMAELLKKYMTEPGFLRWMDRLLLYQLRNQTVEDLKFTFVAQQPA,17.687
+PRSRLRLFMLRLTGMSAKGASPTMLLGLGLLLPPVTLFYGGGVAEHGLPDPYALGNVTIVFATPSVLQHGVHWPIPELGIMALLSFIPIFAPEWRAPTMMAYGLLTGFLLGAIYGPPVVVLPLLWGKIKMWWKLAQALLGASQLYFTIQTAIPLLVTTESETYNPDSRFVMQLLWSHIHTFIPILFIIKAFTIGLQPLQMQHQPGIWALFALTMFLVSWTLARDPYITPDGYFADQKAMGDLLTFNLLQRIPVGNHPALSPPGPYSLLGHISTQIIVAPFIWYWRSA,17.041
+NGYVEQISYHETITSDKLRIDCLLDLNLRFLAMVLKLDIKPLKGELFAAFDCAKMWNCPVERSKDGEPVNQDFVAEAQLRGKVFSCVIIEESQSEYIPCSAPSTVSLEICDNLGKMMPVLRATFQLQLNLGTIMKSTVKESDCRLPAYHLKLECPNENELVGVPQPGPVRKAIDPISLFEELAAHIKFDKNGERKFVQILSYSKKPIKYSVKFDFCNSAREERLEVASYKLEIVSLQEMRKDSTKERSLRTMILLQSSTISFQRLDMYLYKILYLCFLDTKYADVMFRFGVLADISRLCSMMPELKGTWCTGTFVWFIKWAFKVPTLNLGGQDEQMSQFLRYMFKAKKMVIHDPPDWKMACKDSFMPKRNVRLTLCNQKETSTRQALIETKLLEEICTDVDVVMRGEENTVEKSNLFWLVGDSKTVPRNDRLLVGLGQTRNLNASKFEVSHVNIPPGATAVETHPTRIVKLPQIQGALLFYLLYNCWALTPWFRLSKLETVTAITFFSRALYAKLYVTNAHCAQDMLKMCTAVRQLFSGSNFGYLMFHKLITTKQTMKKHFNVQQILSIAVTNVALVVTVGQTECVRPSIFSYVQTVNQAKAIQGVLVSILSPDQAGLAILFIEEGLTRFVLVDYLKMLGKSRQSKPFTLIVGTKETLEEWALYLTGEYVPT,18.533
+SGSFNLTNAIWVRKYKWYELAPLRYLGRCMVMDKSGNKHVILSQVGLLDSPGDELIQGANLPLRITIVDGDDNDFFDQFGEVFELMNLGERAEFNNFVQPADIATTISVQDFIRYSRYLGKGGTFVNEFRDRYLEDGRITEASVGGGFIANLLDVEDLSLVPEREMVFGEKERGFSEAFGSLNRLVENNFSQENGRAEYLAGSNGSSYTTGKIGYVTNWQSQFVITRDLVSMGDFTQKLFSYHQGNIGSYRPGFEKGARVKFGDPIQDWTNGSDPVTSDWSDNYGFKYFVDTPTSTLWRGVVVSNPAIFNMDEIGKNLKVTSLTSYYIKADGNIGRGTKVLAGNNYQVNFELIYFGTTWTLFDANLYYDNGDEWGLSDWTDVVYNSMVAAYQAVDDGHMTISVLGTIYVLMMLVSISFGTIYVYDLYTAMALAASGYLFTRRGLS,17.983
+NDIRSTTEDVLMPVPKDLAGKFFIIEESVVIATETLQKDSMISFHEFGMGSADVYYTVASRPQYISDSTLSLNDTAISDDVTVIKSIGLTVILDLTAYDVSRVTMADRQSYEDREKVSYREIDLFTILVAEAFSCGILTPTYLAERLEQLGRIDIHDGGWNKTINAYELVISASTFADGKNFSTAVTIVPNLPAVGSEIGRIKANDGDIRDALGWVFGETTEQSISPVEYGMILITHGSPGGLLTAKPPLDNSVQEKLFQVLASGWKRGLYLQGGTLVSRAAYLGLDHWLKLPRGLSIIDVSMIDNSSLGIPLYIRYQVSVDQTDKIYEGGKPIPERMDQNRRSFFGTLNLPLAITQAKNITNKSAHNIGQEWWLKIFFTLIRVTVCMCILGFPHTGIEASFMFYLCSQYYSHWFVKWGLEVLSWENVMSVAGMNKKPGFEFALFSDGVILGVVLFTAYIVILFVIMLKRPLMLPIKRMKFIGALLVWSLSVVVGFLQGSPRDKKKFLIKSAIWAVFFSLVAFPNVFLWFFTWKIARLSANASVFYSGTTTMFLSLLVTTATEFSVVQYTVFLEFFIMLTSGILVLVVWLISSQKTSSVSIT,17.4521
+KARYVRLVVAVCLCPFVRYLEIQLQDELEAEAAKKMQLVGREKFNAEKLTTEDLIAVDAVGRAMAEAQMDPATIQRKIPGEVPANLLEEQLKSFLLAQEAKLEARRRRKKLQASGSKSNRVMARERQYLKRCDCSIDEAKRNLLDTTVDALAARSTIREDILLADSKISQLADTSPGIEYPNAFPEQLPYLKEEYYFIRTSRFAFDERVHALQSNLSLLGFDDDLTDATKAYTEFGEAFGMCLEKLDISGILDFLKFIPASSKWNPI,17.6976
+SLIGDLMSDFSGYAEIVTEEYMMKHWMPLGLIDSENTFKYSYQAKMGLAGIENTGIDTSYVRSPAAGKVPVLPARDAGQRLGFTQLLMDLYLNSPGILQTLVYSWMEVQASWMRETRFGSLSNEFETTEQYLPGFKKARAPLEAEQIPKNGGRVPGGDRIVGVFEDSPVSGRSPEEHFQSFSILYIKWNAFWFFSVQCILTLILIIGFLLTVDGLHPCMQPMRYLSLTEFLMEFEGWVVSRRVVYIRDYDFTLTFMEIGNVAGVRLESYHWFLFWTAGLILGSIFFETLRHYIGAMGVVFPTDPPPSEKSDTFSGVTFIAGFSGAMRVALVYTAPQCCRYGEIAADVGHILAGGGGGYDQSCDEYLVIYPLSGGGWALERARKRGVIVFPYNATPWAGILERFLPLVGTARYIAFLVWLISLAVIIYGVYAYALIARKNPKGLMNEKGIKTARLATGWSWFILIKGLINMLPLRGVGTKVFLSQIVRWLPEYALGK,17.7286
+LSFKIFIKLLIYLILIILFILSLFCKTTQTIGMPDLFVKKVDWIYTTYYTFYNDYWIVSVKGVSVEEAIRDLETSFELSKRNVMQLVDAVVWTEASDINPGTDFYHWQWKKLLEEDPLFAKTERLTLVTTFNCMFLAWFANVYALAITTMPTGLFIFVLRFFLLIYALFASISGSGYKDTWLVPFGGAPIRGNLAAPTGRKAFLDCLEYDIVVTNLGAATRATASVLITLFTILRLFTGKWNMIVDVTVRRVSMDCDEELAGATSTTSNMREIERATDVFASVCQLIRSFLDGRNYSQAVANMEYLLRMPESKIMLAWKWNEQAQYPVFRYFVLDAMNEMRVMNQQWMSELDGLFVQGPLRNVFDYLQEQVQQLRVAKQNSFMRFKTKFARGKELWNVWLSKSNNLCQCSDEGLTTLEVAAILLAVCWMVYGFTGTIRIITEDATPKSFTGHLYYQRLHYLRPMMQKIDNNPLVSLLPPRIILDDSTNWKNLVPELINVYIEPLTIPASQQVYELLVVLHHISPSFSGWRRETMVRPNFATDDVGWMKMEVSFINYDQVFYLTEMYPFSQAPFFKLLSQLRIMSQANFRVTIADLSNEIFNYQSLDFEAMKALDHLYQDLGPIDFLFVNTVLVRILNVLRYIRFLRVSRFILPYLRKIARGVFNIFDWYNIVRILFYAFGVSNLLSTIMCSSEPNEDTCDIMQPLDMYLTIFVLDLMLFLSYPQYGFIKALHGFLYHLNTLGTTMFGLAKNNLIYFTLVFSILLILFLGKILAFYAKRNNLEELIR,18.0488
+ASIHFVVASLVATGLVIGTLIGNLIHSAGVAPVIAIALLIIFFCYIFHVMTTSYMNSMSSQGPVDAWTCVGQAIAIGISGFIAAVEGLVATIFFAGLAGAIISPISIYLIATIAIPLTIGLVLASLLVIVLKHICKAALPSVSVVKGISLALTLLVASSLIWRAADSAKCSNCLTASSFVHATFDAISYGAMIEVMAAAASLGEGVIVTAFAVIIALVFVEGLAFALTNIFCGLFDG,15.3219
+KWLKSKEATKRARDKVYVKIMNRETPMAIYTGHHTYWELATNPVVPDKRFVLGEVCENRDDLPYYHWIEHFRSAIDKGARSKEDEGKDRKTSGIYTFRPLTQYQREEDMPTARVQLVCKGVTVEGMSINEIYFHIIHFAADDLNDMAAEVDWGVTEVQLDALVDEPSECEIVTDKKSRPKHIVITTNDKDLPTVRALVDKICLAEVGDHEIQMTRCVTEEQESYIKYLSRHKSDAVLMAGGAISDIQNCSEGRFPITYTDVCLKDDSKWSSANIISHFRGFEEILAEYINEQRWLNGVALRRGFTVQGVSDENPILLITDIQLDDLELAFRQNSINQTSLVSIGSHVLRDIGYFSRGQWGHEGPNQYRTRRHASCWVLNVRHNAILPTEVVIEDGWIHSVFTLYPSAPPHPLGYVQAQWRGFNKENVKDIREAFLKVRDLEWKRHEEVGKLINDIFNTMGYAGNTFWEAHFKRPLFSGLGRIKEAIRRKLIFIRTVISFENLKALVIAAVVTAYLIMAFIILEGKAWGRFEDYGNTTSGWFNLTGHVPRYRYFINEFCLSWHKQTRCREGNFITQIEDKLQACLKFFGDIINSYKGSLFKHSVWGLCPDIICLDKGISRWNVDWSPERTKEICGINPEPRASKSTWRSIQSICDLFNLDGDEYGTYDIDMRVRLTHNSTTPCFPISIGLPCKFDSAGYWTLARIIFEKYSLAFLRRIQIFSPGVAEPLVMVTKGLNTAFAILTLGLAGGLITALYLTFGKPMEGWIESIRILVLVLSLFLVALVLSGVTHGVQYRTFKDDRIKISVRLWVFTRRIE,18.7487
+QEETPSSDRFICKNIVVLSGVAAILIGLGNILICVTTKYVKVLRYPNLRSVLTVVALAGFVANGLLVFIATNGSPTIGVSWLSMAVEVGFAVGLLMCLATTNILADNQNGETGPSDNDFLGSTQAELVMKGNKVAWWPMGFFVVDVYYAKLFAGVNNRILMGKIKGNTWEKNSWNKPGQIMAQVFYIIMTIAIFLSPLLVLVPMHRFPLNVVATSVSVSVLLGAAFTGLPDVMNWCTASFGIRYLGFTSGLAVKIISLILRISGRLGSIQLNFAEKLGVLLVSIAAGLISGIVWIGGLLVQLFTFIVDSFLNTKAASDQPLDIIIMLWFGITWHVLVLASCTGIFYMYNIFILGQSKNYGSISAVSTGLITANQGVELKGYPLASCCVFQITNMKVHEDIKECWTLIENCKDERNVHDIFTITVMHGKKILLTGGNTYRGVEIRVNEAGQVVPNHELYVLAKEVVYSNPRTLTSVRVAKNVELCGLTVRVADKAAMNSMLDKQISNLVHLGMSLHKMEKNVVLSGGQRKRFAIARAMISNNFLVLLDEPTSALSTSGENALFTDLPVKENGTTLVVVSHRITLLKFGDVVIMLAHGEVRVHHLCIYTKLDEFSLKIATYFMRHIGYFLDIVWAFILACIFGLAIFNLSVFGYNPSASVKLVPVITLFITSFLVAINQFFGQSAKGKLAYMHHMVRRDLFGKCH,17.4792
+WNRAAHFLCMMLTFGVTTVSITTRDTYYTGLTKAVKDMSYTNWLIVFQFKMDTPSRTPWWRFENRWLNVPVLASVIWRGIPELFYGSTLVDHFSGVWNIVHWKHRLPTFKRLRGINSDYKPPLRIIIFLSLTFIPDLPRVLIVLGNIPKVTVRFFMLVMTCQPQTDLKQQDGFGFWRYKKPTTANEHNWKELRAADRPLMTYPTTAPKHMHPFGSLLFYGLRVATDQARVYMNEHPTSRAFLNLILALLELIPDPSG,18.5118
+KDFATYDPTALSPGANRRHDTWKPMGTATKVERLLRWGYLTFAMLTTLTHIVILVFVPFSWSVWGNMRYGVEPPEMKDQGVVKFFLLVFSFFLHTYVLFTALR,17.4756
+AFVPITKFYNLRREGTIFKTPELRKMGIKVWLVDLAVVPVAKPGVRASARRIIAYILEFNKKASKLIIRVDASTGFFLTDNLIGFAFKQGIRKVRFITDAPKSGSIIQALFGQHDVVISGADIVGTEFEVGHELEELDIAIGIGAREATRVFAAISACVPSQKIIGGGGTVLEYCATTGSATKGIFLIRGWMEYVNLLPELVRLSAVFSMARLMSTSIHIRRGQGSTPSYAILVGCFVLWIGILAWLVGFFDLSEQEWLFTLPILQLGLAAFAGLGLAIIAKELANITTAFGVLAADLVGGAFCIGGFNAMVHKLPFLYNLTVGIIGLISLAGYIHIIIGGSWWPGPRDREGVLANFFWTPTSNENEDFDILPLEAEDEKTSIDNPSKGGEVNRINLFDDQVLVKQSMTPCGSNWPHLPFVFPDWIMNNLFKAIFWKIVEGSINGAAIIAEDAIARMKVHVKPISYVNESRRFLKLAAFMVHVILEIYVFFCITLEFEQNLFGKSNGAGVPKLLILLMIALFLLAAVGGDITTKWATDVVQQLALYLTPDLLPMWWFETALGDAFECERPGIGTSVQYEKTAFEDKGEPTFDDTLARLIPEVLNVVFPETSPNAVILHWVNFLTFMLALAQSGICVLTGSFFFNQARSLRLCQFQKVTLLATSQDQSADRIVAVLKKWPPEKSGRVAYVNRIFVQLLVDPNKMRVLSGLAFTASVEVLSSVHAKRGAFITKPFTVIFVLLLVLVPLVGGYAFRILQNNFHYLQLLCFIDRDPNLNY,17.2646
+SVSDVDVVKDKGITHHNTIVAAQSKIEIRVMSVAPVTVNQKGTLILDFSNREPNEISVTDKSTAGNCIYAEKHYKKDCVLAEEGGKVRLAGVGKSSSQSVKSKAAIAVQPQAGFACGQNGAGAFQREDELWDELITRNKIAATVALLLGGVMGKTVEKVNSIILLREKESQYIKSIAVQIKGGDKYQICVALVLEQDVLFHGVDKQAPLRNLIIKMKVCNTREMIPKIYETCKDAGKTDVATEVAQSHVLREELVAYTEIIVGVYSPNLLEVVYKMIPDDSVKLELPFDVNGSKIMAVDGKRVLKEKFMFGWAIGNRFGCIMDGKHEKVEKDVVAATLMGIDEPGTNVSELLKYLSYPRAVAAENTLSVEDVNMTMISNHINLGDVSKFKRECLALMDYLRSISSLVTLYNSENVSWQAKTKTRTFGFSFNGNGFPSSLQLVKLVTIIVANYDVQYAWYTGAEESEVNSPERFGCCYKCVRKPIRAGCKTSKMSPTFIILPEKTWEIDDHNLEYRCMGKPALSITLKYDRDDDNSKEDNKLYALAVGLVDSAVTVHGWETFQVSCWIPIPDKKTVKMPGFSDLYLAVSLCFPMDEEKKLRLHAPTLPEIVFVTHASTYIGDEAELVLHILKRNGVCKSLGFEDNHEIWSFIAWISQYHSTNWRHSGSVVCGKIRQLLQDLIPSADQDTQVQAYCEECKNQENANIDDSTLMLVIAYKNLQYLRAGILPDYYTFDNIIQVGSNNVISGAAMHFLDQIEPFFVANTDPQKNIQLIRNKEDEFYWRFWAFDAYANEERTSNNRDIAKFEIATKTIPRLQYERKYSEALEVIKGFSIAYNEKY,18.7321
+GENFEELFARVRRQHPEAFFLYLPVILIIGTGVAMELFPFAKKYWRFSSALGRAFLILLLSILVLKLLLGKLDEFRIESWALADFLHVVQAKTAPISPTIAVLRYFRVFRALRDNMLDRTHDLSKPVIGYLFMAGLPILLILSTAIELGVIQLDGLTILPLLTSAAFWGILPQRVTSGGDGSLLAVLTGFAPSLKEGFRYGFIIGLFMALLGFIYTIAAENDNEALRQG,15.1469
+PYVRSGNVLMAMPQWLQDMKKTLSSKRSQKELVKDGDRIGQKLVKERKMSTVAMDLYWMSLMLAHPYAHPVLSGQATVYHAVGDGAVVKVHDGDTLFGVALYFSENMWFAFFNFAPGMQAPNVSSRDGSIGLWGHLLPAPNFSFAQLMMIWFVVIDFLAGLSRLLMLYYYNLAKTFRFHLLFASTVSFLEVQAVATSFWAYSGNSALLVMVSFLITYTGTTFLAGSMHATGFYVIHLTDILSQHVTFLMTLIEAMNSHQSATSMSGSRADGKTTVNIMLYSASLLNFTTFKGGFAKFMCYISALLWLVILLYAFVDGAACIGFGGRLRRFSHMAIVKDQSWKFYVTQKGPGIIQAAEYMMNGPNIAWSFVIHVTASHHRGDIIVSGGWIGSLLPMAMQFAGQWAPLIVRAPKNPRVLKLYLLVTMLPAGILIAITVYTLWQPVKKRPQRPSNESNMLIVIVGMALGAGTACLPFVLGEYNSNIVVAWAVSPVLVRNCFIIFMTVPQMACMQDTICSVDRGEHVTGLNSLTVVSVTVMSMPSYVIAVQTVSVSKSMLGIPFPFVELSLKADASLEQLAGPINIKDTVLKQCGAVVIILMLVFGILRLTFGGVGVVDLYSPKLLSIAEAKVVFLFMTITWGSGITNSTKVFD,17.7269
+KWAQGYAVLEVVFTVPFVFILLFMFITTCILYDAKTDFVEFVLSIAFFLTNSGIEWKVCAVTASSDSQLLAVMCLVGLAYYKIYDYDCCDCPFSIDPKKREKTVNLKQCSQLIAFELPPKFVAREEVVVSQVPHRFKNSEIEDLTFELEGLIYDHNYPIEDGFEAWRVHFLVDVGGGEIGELAYPVYHAPVMSIGYISQRPIGIKAIVVRNQKDQMAELINEKDVLSISYSVGLSYELNEYQLTTIRNLRNSAAGLSGLKIAVDSIIGLCRTPGLFPFNLSHATSQAITVVLGKSKRFNLDLSKIKGVPALKSFARYAVKSVKRLIADADSLPPTLVAVFYKTGNVSTVKGLSPPLIKLNVLSDASVVPLGKKINGIGSTGAVCTIFNGVCWATTIVSQDDVPTVVVQITQFRLGSQLDRWGKRAQTSDDMFWDTAAGVRLIVQMGVGSPIATIIVAVRPADMYNNMVENSEEKLKLRNNRNRQADDEIYVAIRMTGGNARQVRLGLSEVQQKQRFVLDIPTAGLIFIGKEFTSVIAKVAGVYPTILLAERTPEDNSVSIYLRNVNYIKGRPTSFLGTGFNNSKGEFLDPFFTLDPGPQDAVNGLRIAKEPESHKILEEQHGPPCLTYNQHESMLQILKRARLSIAVPDNRVAD,17.9063
+KSVVILVGCASSDPSDSIEFHFFGDNTAITKGRIGRRRFVVIGGPSADLDDEEGEYGATHVTVFDIAGSIMPIGFTRGMTRLYGISHLTEKPLPGGFVMVLPPGGWRELQNLQFYEAEKYIRLSESVMQDVNGGLTILEDLGDEIIKTSFPFGQPPDKG,17.3791
+ESNISKIFKDPICAEFKKVLVSKIRPIKTTAVLAGLAGFFCGGGFFLGAITADVFMIGTIVMVYFAAVLKMSDARGYAWFFSFFSRFLIGATNFADFGELIRAFLKDVNLRKEKVHKGNYLALFGVFGITWIVLIMTSLLALGEFIFMVGDIFKQSGKKAKAKLNAEETIAIANPVIYALIMIISFLLSVATILTSSTGARAKRIQSKRRNLGVVLVGIFALIVVAILFILIVELCTSIGIQASYSLLAERLIGTSEYMEGIPNTNEYWNAQGVKQMLGVASLWHTKIYEWWNAIFGFFVIKLIKFISDQAFRDWKNGLHSLQIFVGLSVLSAGAGSISSILVLSDIIKNANTGSFIIVPVVFFIGLMINVAIFAIYYGD,16.4909
+KTYSEDMTFLNTPIDRYDKPLIDRVPPEHHTYVRKIITVFLVSGILAVLLLMWATPQMHSKVRWLEAGNSPGVGRIKLDVRVPEIHPQTMHILNAILRFTKKQKDGPVLVEAKSDGDTIGTADEFAPLAAIARIEQDLKASLVVRQIVPHACSVPYPMWITEGDKAWDGVFYKVEECDTMDDFVRILNFMIGAEYLPGSNTTNEYCGASRKIVCFEPVMIRNGDDAEWKASVVVTMEILELVMQQVITCTDSAEDGFLISQKGQFVGEGELGILSVNLEKQLYKAVEIRSQDDRLKTLIMLIVSFIAVAVGAAMVSGYIPRRRYQVTVNKVPYRLQDDAPVEGDVFEHGLTEMRIPVLFSLVDKLECVAAIDRQFKLRRAERALVSFGTYLQQGKSLAQSWAPVFFGFAEYLKTIGVCIIDNVEGKYLKNTVAATMTLFLLFLCWLTMCFPLQHPPRLPLADFRYLRNLPGGSTTMLTVFIYAHGLDETLDTEKGFWFWSDTGLLGTNGNMSGCYFVSFAVETQAFVAMMLGRSHSLSHRFHGLLQYTTLGWAFFTSFIPFVRDRNFTERHYVIPQHTKASTITCQNKSVRTTDKPQARKYQEDLSHETTCHCQTVEKTFNHPRVKLTTVKACGEVWTECPVDIAYTLASELFYSFDLQGGTLLRPQFEHPWGRSNLFAAFFHMDEGFSCHLGRDPDMQEYINSSSYLLNSTNVSEVVLLEFCDNIPPNILLITATFFYGNMMGKDAIAIPYDEYVGKHAYELAPEAAVIVLIVFAVKFLLRPLLLKCLWAAEFADHALNRRSINTPTAFPVIHLFFDVSTVAAIIGKQKNYDRFFPQLAVDLEALVDEEGKEEISFLIREMQRFIDGVMVMLFMRKWRKRRTLAQLRPPAVSSPDAPRNCLNVPDREKACSLNNELKTNLAIAADY,18.5435
+TDAIKVLVVVIVGIITYLLVMWYWSGVVFEYGPVFIFFLLITPFTGEEYNFVAIFDAIAK,13.894
+LGRRRIYGGLFHVLFFFTAFIFLVQGLRDGGRLGVPPHTLVALFEVCVILVLSASYASTFYCDYLSTTVHIMHIIQQLLDCYYYHNTVLTDNNPWAIDPVTLSLDKTYEEEEDERVEDVTLNILKKQYWHQAFQFEKCRMNMRQEEEESWLLLGNPQKQCESCKVRVSDPIGPDSPRPDVTIRISSVDISRVLLILRGSESLACEDKVRFSLYRAYQEPSPLFAEEFTEDLRCIVHTVRVEENALDTLDASPELFYLPSCNAQLIACSSFLRLNFLLKWTERLALNDHFWANKGNLYHKSWQSVKEHEYVHFPYCLVRLGGQDLERVEAHEKKKPLNICLSITVPIYWGDPRRKEFCKICLETDYTGYDMFKKRRLAMIFFLMFLFFWVFSLYLVEHAGQAMKNKGKAVKLKEKLDRLLDCSEMKPKPIRDGNLKMLQMPGTFIDCSSDKGVLSEFVAGYLLIGVFFLGFLVYVSCSCTILLWLYWCIVIVWIIIMFYWWVSAVTVDLQMAKFARQRERKVVAQIELAWRLSLFWVLYPFLLLLFLLSYFGVSPLPKKVKGIGRAKDHPKLFYQLKILFNLTLLLFGSLGFNDSMKFPWDLVLFLFIEFVLMLYFFTVANSQGFSFLYRLTKPLRNVSALLIIHTFASFLSHVVKRIYESALFSMMLQTLVVSFFQQYVVIIYQVTATNFISRLIKTLKLNLPTFVITMVFAFLANFVCKLLMRVNNENYTFILMAVPLVVPSLFVPFTTLGLNSVAMGYFLRGFFCETLQEKAKQSVVKFKKDEPTSRYELTPTR,18.1273
+SEKSISRALVTTMRYKVKFTHGQAVFERQYRHVLDGPFGERDVTGKLRLPPDPDRQLSLKNVYIVSAHFDPKGVEAKNDYVSVSDSIAKRAACVIADLRHQGCRIMYPGTAKGVEIMADSDGVHLQLLLQATKGPGGKKTAIADLRVPLIDYDPMATIIHAHVQGGPVFLREWTVIPVYVQLKFKNDNKVKINFIVPGDIQPPLIQDNDHWDTNRYKDDGQYKFDLLPEQIEIRGGYEDIVIDPGFTNSFGRVLTHCNSDAVERRTLPLPLWKGLYTRHKARSQDICEVPVLDQRVVLPSSRVRSIPELAIEDMWTPSLSDSVDKNVKHAGNKCNMTNMKREFTGIRPGDFKQALLGLTCHTGREMNINCLKSVLKGNKAKTMVFLHGPMTNTLGALEKLNKENPKRKYMVFKAFHIDADGLSISLMISSSGANLPYSTGLHLLNPDGQIVALNVIAPISIGGQSVEHLQENLLQKVLRNNKAKYSINTCVLSIDWMADLTSRPTKLLPRLYGSGYSISDIVTSPSDALFDIGAQEFVAAPLMQGPIDWLRAIAIDEAEHEGMTMSNVVAEAIENARREYTLGLSGVDTSGIAIGHARETVGREGTASMPTSSAAAAQGFWWATSILNLPPTMTALSDMIGGQVVNGGSIVVVGDGVSNIDEEQRIMAQQMVEQIIHLMSVNQVSALALKNLIEQENNTVGQDMLMRPDQTAVSLILKTSAISELCELTDYVYQHSVVQSQRILVGPSGTAIEQRTSKAALLDKQLTMPALYMEGDAG,18.4702
+GPVSNYTYTRRDGLRAWFAQTEPRTIAKPDPADYLPTHLPAREKLATYKRVVIERITNSMGYIEYLDARTFNYISSPDANHIIIETVKMIACMLFAIGIIFSVHDQVTATQRTAVALLVAIEVLPNGVCPSGTNHPSVFQKILTGEGIFCADVASVGAARAVFITPQVQGGSLLATKVIMHEQPRPTEIVRDPILNQAGVHALCGTKVEGDVRQSANFTISFYSYSSTEGINYESTQSDIYHDKSPSNPITLLCVRYSANGKRDLEDGEVLSRPVTTQVTHESDGGTEKVRKDNITDIVIVLEKAFPAAIDEFRITTIKILIDHPITEYIVVCNDPNPVGTFRLAKYILNTYPDGSVVVRHENTFMKMLSSIFIHVNPDPSRLLNVIPVTNSLVKSGRYVMGDSDVVEKDNMKAVLKPLFEKVVGSWMGNSTFAMVAGFASLASFVFAFGHVSTQYAGGIHSEPLSILFGVNFSLSTTWVSAFYLLAIILMGILAGLETLLEGEQA,17.9375
+IGFNTTTLCVCLMVALAITLTYFIKNKKSAYTIRLECTQASNCEVINFPRGMTTLNSLPSDDIQLFHTYSAVRLALCLGGSLILGAVLKIIFTNTELGRVLHAKMLKNGSLSALAIVTIFVVFFIAMSNLALIHAKGSNKAAASIPCGFLTYSVLALLGVFMSKCWNRQMPMLYGLSKGHC,17.0854
+NGQVRERMVVLALKDPANSDRINDHSMHIESYTFVYYPAQGGCIHVGIIRLRKPQKLSLQEVLAANGDSVILAGLGVDACSFPDHIMENFFEWGDTQQKMDPRVGPNAFVYDQAFLDEIEGHDLFFLREILDSVTIGNRLQNPLQIFRPYAQARNKITRSHLGCSVTKMMLRNLETVCTASFQTREPELEPFLQEMRADNVSPVLDLLEEFSFKIPPGANIVWLAPVAWPILQMVKRQLRGTGRCPEVNHVSPGDVPKGATQGTWDALDAAQLFIGGDSNIRGSRFFFQLCGLVRLFRTARVASVFTKADPLSRTALPEQGAGMERLVADIFESVHENERVANLADLDRELCDYPARVQAEEWARACGRAKSHAAYLQSGTVDTNVKTHAAHYVLKKEELDFAMGFQGKTLALSGHRCLVRKRVASTKPEIFTQLEQLRVNNPGIARAEYTELFAQGSFVVVVALAEFRNAVRQVQDDSLCQELIEKAQMFPRVLQEVRKDPTPKRLSFTLTLIVVGLPRSDFALLNDAFLACFIPNPRGVANVILDTNAYDLEDLTSKPEVIEFEGICDLRGVAAVAPKTPIPAPRPMEGGKMFLISDMEEDGKGYDVPLPYALGAAVVVAACDQCGHPTFGTNGIAERIVIYVYLHFPAVVALFNPIGWACFIAHSCDPAFNSSFCRLGLPMLIMVVAAIVSGTLVFTLMIVTETECLDSGESKQKALAGDMLIAFPLMGLLELVLMELAILKGSAPSSSRHVHKDDGFVAMSPLNGLALFIVLLIGTFHGSTTVSGQVRSGRRTDMGLQGITGRAAGVRGHHFIMLSRADNIVTKLVPPWATALMLLLLPFLLIAEIEMGAGPLTLMDGVRSWISCLLVAIATAAFFFLIGTFQWVLGYWHRSNDSFISILTALYLLINIAKLGFGFYLH,17.9518
+VPISYDIKVPTGWFIDGDKVRYKPCQGLKVITLVNDWWIILEVFACVTLPSTLILEKQDYFHKRRCTTIFESVAEFTQAYQVSIQESTQQHLTMAVQQVLGIRNRVDRKYVILVANDSPVVCYLEGSKVLFTLLQGPKPSSTVLLGRNGKTGLLLRDVTYMKTNGFDVVGGDLSVGVKNAVKAGFYPVPLTQIVQLSPVLTQAFFDDESVTVLDGPIGGHMSNKVNSQAQQNWKINNDNGFMVKREARYLGVTVIKNDLVRGFEDLTDVLGGCSKSILGALEMRDASELNHKGISVVAETISNAMTLGVSELTCCTKHRKEVWLQRRLRGKWLRLMLNIVFAWQNDLFFTITFVPNIVHFQRKIFMPAEAVFNFLIASLLFVLIGEFGFFDVEWRRWHWRRFNMIFYVERYFLFKAFLANEVKRGIEEAKKLLSLAFTFVLISALHLVNRIVNLLADCSHSRFLVDNLIDLRFIDIYCTKKYNHMTFMLLLLAATIITFLLIGINAAMVCCARNDQVLQLIQSLESLFNLIAHLNYMTVNKFTFGLMLRMNAYSLLIIVNSLTGYAELRIASAIMLRLEQAFYDLMSRFDVTLNGVIADRVGVASYSELAVAILQLLELLVMEIYEYSIILGLVAIYFMVTIGCCVKTLKFQGLDAFNP,17.7417
+IFFLLSNSQDAYADKFKILVPLLWFLLSVGFAVLLHWKQSIMKAIMFNLSLVAFLCWLVWAVSNFLYKLSDIKMVFCLFIVVIWVTALWSLSTPISYRYTVKNFVVNERITGMFPNLILFAEIVPAITYIYFLFTFLDFTYRLQALNDVTVLGTKPMRLIQVLLHLRVAIGFLVVNLVGTFYDETMEGGEVWELFNSLTPDINSSVTVIIAVLFVFFNFFFVVLTPQHTKPCKKADNSSKPLAILVNGVTLILVLSLGYFFGSLGICAFSANSASMLQAFSMHTTIIMLFKIGVASAWGQVWNQRTDLEVTDHDPPLFILTLLLAGWVIESTTNFGGDSEIMNLLGFLAGVLTSIEIFGLQINLPTFGSFVQDWGAIATTGFQSQEQFFWITYGLIFVLPVAMLFFKIFHEIIEFNTMLIICTALNASFSLLHLSRALKFETKMVGKKRCSADEEFGARMEDAMDGAYAFFSKLLNTLFVAVFRVIVIYLVAFFIFKMLWIFIPTIVDKTNEWSVGGFVCELLFSIAGNLMGIILLAAPNFKMWFLLDVNTVFLFVGLLGLINEISGLRLWEMRFSNWFIKFHMWFFLPLILLLPSAIITFSGFIGAFRIEIVYVFSLIGLYSVPLILSALRQR,16.9356
+AVITQRVIGIVAVTLLLLIALAGGILTPMGETGSFRFPEVSWSVLTLLKETGWGSAEDGPTLQLGRLVTRAIVVMVFAALIGGWIGAILAWLVGRRS,12.8882
+FISRALFETAVILILVLSGILQGMVLHSSAPITDDMELHKLLAKEIYTAFCIVAAYLVVCVGYPLEREDRLMIGVYGSCSAGPLDNVKEWGYRTIAPLLVAYFGLFTLWYYRVFGEDAEKIWIQTAIFVAAVLGMSILNLLVYLGRFPDAKARLSILLKDLAPNVLFHFVFFLINAVALMPFIYALIESIIQGVGIKKFIIEQNGVNVPLTIARALKIRIIDGVQITTPNQNIRRVSPTVDDRMGPETSRFHNTLEQDDSIVFIFLFVQDLKILGLGSQYFTLFINVPAFYYVIDIINVGFTVVIIFVVIELIKGRFRSLVGLFWVGSSIKSSDFLAGIKNFVIFNIAEVPGVLITTIVEIAWGADFNSAKITIMDAVLIVFWFPILDFAWTNIAFATTGNFYFLIIVAGLGMKKADPMLLATLIYAVLSGACTPMLEVIMESVYVVGAVDAIALFIAPMFLRFNLPILVTYETRRPNLIWLMALIYFVDSYHLYFKSWWFFILSIWGGVIGMDVVGLVWILGNYSTVSIIGMG,16.8602
+KATQSDKTFPLEVSFGFTASSGNIVDAHAASMATYITLQAVADLVDSPTECPISKDISTEQKIWDCLPTVNTFTARTGRQAEIKSTSLGQNLGLPYFDSSKSDKLHVDMNADGRTISDTLFLRDTQKQMHDSRKIFLPNTAAPNTGTKDLIDEYGEGLIVNHDTNDASGYLLTDELDCHRPTMKAGSLNPDYPSILRGLKVNIEDIIQDENKVSGFYQVLSYLLSKGSRLKRTIKFCHERDRYIHSDNHKFVFSGIGDQAKMEKELDVAKATGVTIGLEDSMVKKPSTKGNLVGIIPMNGTFLLVPQDPLSGQYGSIIFAHCIGKLDEDTAANIHTYYKAWVMMGNLSLYDKFMSAKSFSALKVQLRVLIARAGYIPVLQVQTNILDLSITEDANIFEEVLSSVSNPFFGAKKTIRQIDHDRNVGFSVNPRGIDSWRNDMPIVLADVAGSLTLTTLGLGFRAGLSDADLCYYHDLLLEGRVDNEKNAVPEAKARKVSQTRAGLFKRLLLQMNGTDFSIRGCQGQQLDLIASNGYQSLKNLQACKKNGSVTLSISVVMRMYRFVADFAKNNEALNINKYDFLKHTDVWYYPGEHNRDIGDLNLREIKFKPFFTCDNSPRNTVAVFKQLYTLPLDGRYFVSTKRETSKLITGEIYSISEFGEHKGWVALANENPGLVSTIRQTGRVVVQFMAGKSVKDAPISKRIMLVPCDKVIVMYKSLFYRLPEQIDTCYESEEFYDTKKDCIKTALLLARKIGYGMNALGHDERLSALQDVLRTLKPCDVLTLKLVTDLGQATGNMILLIISVHRSKLRKVISILGNCEAIARVLQTMKVGGVDAVILLGNNINDKDIEPTVYVGYSIMKEGINVPFTGGINNGAVLACMIKILEPIPIFVAPARPFRYYLRALLGFLDLGVGYLSNADEKATFASSYRPGMTAKELLAQLAGELNIPSDLPRTMEIVKDL,18.0526
+RPKTAEAIGLLRATMLFALILAFIALAELLAGAMGLPGSWVGLPVPLIVLAILLVLIGFFLGLWAIGGFER,11.2268
+AGVCRVPEHHEMPTIFCDRDGNNGPILADNAQVMAKLEERQQGRKIRAYSSSHFPTRSVFLRMGSGALAAIQANEGNVRLLEFDSSEPRAPLSKESDLFGYGGIGQPNSPVKNDYKQKIFGGMIITGPPSKVSLSLSGYAQAQPTTSLGTLYGDAAPAENSKVKSLLAEGNTRWRFVDNTDDATIMAYGMRSIRFERPFGMATLNAPVNTDLAPEGRAIIKCPTAFTFKFTGDDEYYTHSPPAYVGAIEDMKKEKYRPPVSLSGKRPLVIEDKRAGVMDIYSFTPKITVVPGGRNDVYYDIDIRISGGIEAVDRQPMFMFAVAIALCAALAVGFVMAAACELMGRSPRKAGTQYRKKRILVLTSNTLICYLLSPLLLEATLIDDSSCDCDLDLEGNRFDVTYGLVSCDDLQFQTLFSFFFFNLIASVIFVELRHARRIYIIEYPLFESPMSLVGCVFLALFAILLPDTVADGSGDQPMTFYTDIYYGLTSLGEMVTAYRLINLATGFLIGIIVELAATLFIMLAAEFFRSSQHNSADPRKFQREQRSFLRLRRELLPTTSGVVPTNW,17.8225
+NGKNLAERFVWQQLSAAPFTMIFTMVQIGSPDVTEYGWNFDKRALAGVYISGQDVRLIGGARNKNVTVTIESVVIGGYGRPSSTDLAGHEVLAYFSQSPARSRFTNIYARLQNTRGGYTAKFTSSFRPEAQTYETGALSIRFQGTADAPSHLQERTIGQLESDGTLTGDEKSYRTGINRGLIGKWEDAMAHTLGAVGTSGSALVLISGNHFGNGYAFYGAGNKSLTSKLVYDNPFTQVNTQERFAKDRYPDLTGLELLPENVQVTAVGNTSDWLKGSIMFAAGALAGLGSGQIIAGFAAVRNAAEGLGVALLIAGGTVVGSKAN,17.1103
+AGLFPNELNELRRRLASDATTFIAPINFKVMLTREFQLLHLVFGFAVGLAWNLLMGQNWPFFPLIHGSADDLPKLTSFGVIVHMHEAVEPIWAWYLSLISVQIHSGKFLQSIANTRLVGSLNGMFPAWQGGKMIGRLIPRHKIAATLAIPSLPVLWGVTHIDLMPESLEWSMNLVEGDIALFQETGTFVDIFLLAGGPRYLFQVTFDEKINRLMRKRPILIVSQKIGSHHFKDVEEYAIAMRQGIHLEEAEINIPGGKVTYTPNYLAPSYREGRNRTVGIWQTFLDEAWESEAELAILHKDLVISGEPVLYPHQFRQGGRELVGKFFRLVTVDPRAFVAMQNGAISKEELVAPLITARERTSWYIFGIGAVSATLVATGPVNLNCTQIAYGPSLSAGLAHGLIFLLIAFHLYHVLKCSQAFQGLAAIKALNLIKPTEQALPERIDLDPLVLFNVGHTLIVTLFLYLSILGRGDVGLNLTMAGVVGVMTVLTYAKFRHCPIQSQSNLNMDRADQYRVIDGLQYVLKEIEKDFETATGL,17.607
+WAQKLIILMLSVILGGLFYFSLLPLLHPSTTLRNAPIIMPLLVLSSIIFYWFIHDNMIHHFDWNFKEFRLISQAISLNAFAFGLMLGTPDTLKPGCQHIRNPSFILYYLVYFTWYTGLLAKQQKELLKSLWSFLIIDSPFNLSVMRITRANLIENFSISQGNYLGQILLILLTHHSPLTILSWRGSPDRINDHASKNVPIKVDNETDNGELDKLACGALWSYYSQLWIETMLYRPTNGTEKNQYKDFINLVNLESYSTNVTSDVREGSPKAALLVNELHTYVILNASVVLFVTSRRDYNSLKKRNEA,17.6565
+KGYDIRSNASWLVTRADGKRANAVAEPSALKPGPQSGVGNILPKSRASYFILNNIGAKIIYLLDILDTATVGALAKAPPANTNRDNQAKFKFTATATGGASFSGTVPTDIVGIKVGPTAPILWGQVVGGGQAAAGGTKGVTVEGGSGYFVAGFVLDDKENSLLPNSENVATIYIIPRGNIIVNNISEKTGPGVIIAAEGLAAKGGEMLVARGNSQSSTVVDVSKKAESKSIITEELLKTAQGNNFRADINQLVTSLVDSWDLGTEFTVGLNNATPAGGIFASGGTAVNGKQSNAAVAYGGVQIPQNGKAYGTMVIYKGSSQSKLDAVKVRFSTTNAPEYWFLVGAQDQLAGNNTGYFTGKNALAFAAALAQTEANKVVLAALTNKDNPVPQNKSGVVAKGIAEAFTEKFTVDAVGTDSIVANFNTKLAPGQIVFIGPDLEITIAYNGTVLSDAVGNNAGEAAILNPNISKRIQEKVEIGFSPKKNIGEEYISANGSIDSIGKCAADETAEFSALASTFTSVDFLVKSEYSSYSSNNTFDRFSLRDTSFTDDSNSKNSTRLKASDSSKFYDNYKMVKTAVFNGVATPAAAALGSLSQFTGTTTLRLEFDKGAASLKGEKFSDSKGNSVTEKKFQDTLRVNTLGRGVAFIGVKVDSPKALVIAVAGALGIGGNVLLKGGKIVASSKLKALSAKQARNQLPLFGEYNFLSGLFSNGATVAFIDPLGIPATKAIIYPGESIPIEVFTKTPTRVKFLEKGAKLGNTLSALFVFTETANLTSSLLVRANPGVAGNTKPKNLSDTSAGTSPQFAVTAISHRLALTA,16.9213
+VLGVAEKKDNDDQAQSNSSDDIKKADRESTLYGQISAGVQVGAVGTAQQQVTFQLGYASLWGNSKWYHGLNKRDAVASGYESLMGSMTQAGNGISVRGQNSSSDHMSSLNNNSNNQYAGDNLLFSGGNVIQDMGMAQSLSYQGPFSGIQYSSQSYTNTNIFWWSGGDNASDIKAKLVYKAVGYDNYLGEVPGEATIQVRNLKFANNGTLAYAVHSQILLNGGKVAYNGRCMVSNNSTVYYSKTLQSALAQTWYEQGLVDANTLLVSAQGKKSDLYSLAKQNIAGNRRASFAYGASANPSAQVNASLSNTFTDTYTYFSGTPTYSRSSFP,18.6621
+VGIIMPDSAAAFSVAGSLDPLQQVADAIEEFAEKKLSHFDIDDTSFINIVSYTKQVVRALFVQPCRQMIQPFRDPYSEIKFVNLDISKQLMPSPRRNSSVVKQQLIPFGKVWNILHKVGLNIIFKDVTVVSIALALAANLIKKSEFLSLKMAGRSVGTEEKLGFMGFIFMNTENYTKGNIPGKEIVAMYFLYLQNILFHPPEIGSPEPAKYSEQSGTYPCADAAKKYKKYPLQEKFMFIHASIGVGDVGKKVFSQRPEKGHLAEMLGAAVLFFGENFPQADFNYLPSKEAEYNLSLALFKFGTQFVVNNQPAFCYNEEGNGWLPVNKLESNEILDCDTPTKGTKVSGHLTPVTAGWLHLLQNLGMMCGSRQAQWQVYPFHNATCANLQHTKISPMAGLGEAISTGGRIRPTYRKILLGIPEDHYNPSSDLNMIESQLVQLDKLKEYEAYHFGPQMYAQNFPEKTQLTRLMQLSILLSDDRSLARLNNKSLQKMTESPEKNVTKSVPSTITFGYASELRRKKDRTKNTWVNLRRKENGAFDELFVSQESIHSNLSVAEFTIDFKRNDGVEIRACGVLCVTDAFRVKSQFKAIHVVGMSSTTLAKVNCQVLSPNNVTFDVNNPQYLHRQQSVLANASTWPQFRWQGEVSGPTLYQLDSIGVLPKADPQKPKWQAGIMFVLKLCLYLILFCYLPFIGVMYLKPAIFEQSTPPYKQAEAMHLLICCIVIMLFYSLSPVNLQASKQVEGSGVNLLVLFMTLLSYWSNLFRWFGHLFLMLTIASSLIYEAVAKLKTITPKNLKSIWHIETWQVFEPFMVFYIVVYSVALMSLKVFVDTWWVTFFYSGPVIITGSILGHGVNLLDRKMPYSSNIKLALHNVHQLLLNINVMVDEFTGQPTSPFVTNVAEPAKRAASLIIIAVGDALLAYMLGYTVPLVPRPK,18.4627
+KILLGSSITQSWLTYIPFVFLLVIPLFMIRHYGILMTNVLTILILCVGAKVLANSKGDDPTSVRNLKDVWQKAFATALQITIKYYFGKSTKDFLQSVSMIKADSTIFVRKAPSWPFNLSVATVRGATTNGMSFTLPCTGEGNGGFCLLSQEAYAVTGPLLEDVGVLAPEGAGKLTQAPELVVGKVGDVDSKALLSLMIHLLAKIGVATVALSLIKGELEQLRVEGTDIARELATKDSKDDNKGSSLATIINPPMNIIVTVSATKPNNTVGGRASRALTQYLMDAGTKVVISTSTYRDLVPKARNDSSLVKTILAFKEDRVPLEGIISKRKDTVELRVVLIRLIDRGRIAEWLDKDVKAIDSSDDVTEDLIVDSKPMVHPMRVGGTIRSDYVRNQTIIHIYEKEWDDLSRVIEEKQEMKEVPFMWIQSGKNMEDEILP,17.3175
+MFFYSLIRTAPGTLPLRQSLIIFVSGGGGDGSVAEAGTSLGAPAAEVFHVTIAARFSHELFATILLAYCVASGDTLSKVVSDRAHLVQLVTHHVGLARLRMLVTVIHFTALCPFGGAILFTAPLDINTRQPDPDPVALWWYYIAPVTGQMTREFGGTIINPNIANSYHLVYFKVLFRHFVAEYVGWLHGPGMHPTDVLDIKAALKKSPTHGPEIPHYYSPPRVPRAIPPIFSVFNEIGDARYTTIYDGSVMGLLEKARTYDMEEVYTPRQVGYIFVHSKGHNVFRLVAELESAIGDLFTAYFDSLTSEDGKQQNMISAYLKGLVASHGCGLASAFSFGEQEKWRNAFNYLLWGRYQIESWRTVEAIGPDLLSFWRKRFNELKQAGVWITTSPTCWEAGSVKDNGLFIINSMKYALGRDAVWSANMPRVNKHITIEVKGAAEHQQIVDALIALVKDYDNLGFYSAEERADHRFFVAMVKEKGSAGSSKIDTARVWDVHLIRSRYFYYDASAWYHSAQRMTDPIVNRGYNIGLFAAIVAAGMLLLVLRVDRRKITCPFRISCPDERFSLKSHEIPLDGNLRVYGELKSELDHEDPFGDLTVFRGTDTELSSGGFPLHWEFVKEPEIGMLETLIQAVVGVYFTTSLYPGYADEPGRTEMGLYNMGPFGWWLVKYSDR,18.1742
+SNTARSTQMVGTGIDINSTQMYPYNIMLTGFEVLIRLTPSAIENWQIRGEEALDSFFTSLSNAIGNACVTVFLMRILLAVYTTKSSAESDRAIGYATAGLPNNITAIVAQINAVVATAVVNSMNACLDLAPILYWETLQKIDNISNYYPPFDRDCLKARAMTYQPQEVRMDMPITVACQSGRLMNTAVRKETVIYAIIKEEPKNNFYLLTDPVYQRADTVVQAQYGHEPEFDTEDNLYPNKYGWIQYHEEYYEPIWWRWKIRSYFRTTQLETKTSLLARDSWEPFYASPFSRHIPISITDRPGMDHFMDDLYQSTPSFLTNAICCHNTNGHFPTELLGTIDTLRSALGGLDLHQSSHKSHLLLLRSTIRDLCEASGSGMTTQFTYLLLGNVYIARSDNLKHAFDANAKKGFKLRVIKGAIPPHVKMQVVIGASAGRILLMKTSKLKFVFSDGNLQRPLSEYVELAGDDISEAVFHAGKDTFEFEYEVTDDQFFFHFRAELIEPWKRENLYDNSLYLFRIGDKKFVRTLFTTLSCNKSVMLYFQKFAKLKITASKKKGISFTPDRPACGISVVPHLDKQFVLQVVLQTLMKITWKPCKKNRFQSRFVDHGGFFDFVTYSEIYLKLFAGENVVFTRMSWYAKLSTPHDFQPRSLVGVSTMGIFDEADGKYHLIGTGNFGFKIWRFLYVLDSVFSIEGMFAATITEYILWSGIVRYFRTFFTLEAGIPSHSSGTEGVYVCFKELIFEWPKDTPSVQISLAESTDPSAGIWIREIENRNQFNKVSMLVKTAVDVAQLVFTLEAFAPFEQSLNVIFDNEVDVSLTKALGPTASNTYESSQLALGNRLVLSEAGDVTALDRMVTTITCNTLCFFRHYNGITVVNCIEKAAAVVCIHITNPMPGFVQTQLGIGGLGPYICKSCSLAELQCEDRKIRFYSQIPGSAGDFFEDIAWQKRLEELKNLPK,18.5522
+LVKVETEVKVYVRPTKPLPYTVETAYGGSPEQQFYNIRKLEPGLFADMGGFFTPPMSTASLGSTYQIFRQIYDGTLLWKNNMSDTDPNNVYRALQMKDTVSSLMFVLVLPGSDVYIKLGLVHIEETTKIDGTPDDAQPSSTDLSPGRFVEEQEVMSEDDELALLEDLRSLKFVCQDVLKRQKRHIFNNASITESLTIAFALNNTDNRLSWLMYLWIFALFGLVLIILVLVAGFDIWWSPTKQYGMIIFNLIGNFSSYKTMSEASMKSGIANVGQAWTTIKTVTAVLNNKALVVVNAGEDALALLKSLPKQTDAMVGHIQMTDRAIGMKPNDLWTFIAPIGYVGKGTRMFIIFPKSILSPNPSIGRTVVVVAGIKDVLQDMIQISKTKTDKVSGVNKGDQVDFRYKVETSKNNSLVAGAITEALRGSSASGIKIGSFVQDPNPLFGDLENNFAYGASAMLFDAFLTRFNKENNLIVLVGNRSALNTDTRRLVQWVDALHFNTQLFIIAMEKNAQVQNALTTANKLGILKPTVNVVDQYFPQGLLISLDINRAIHASLKGLPPKGFVVTVIGEEPDSSQLVAKVKAFGLKLFVYAKSTSDVAKLNDLGFATLETGGSLEFFKMDQFKLEIANEVTKTMRSFALIVVLDDDMQQGTKILTDIHPHHPQFTPGPKEASLEKKLALVLGMAVVYRLRLTTVRLEVVTRIPAVIVVNDIQIFTDMAYTNVSGNLPRLPADKVKLGKYSYHAADADGINYKVTGDGSKLKGSIVAQIMVNDVVVLNTKWPIETSKWKIAVNAEVRLDIGPFSTNIPRTMTGYESYLGNRIVLIGRKNRVFGRSTIVEGLIGFLFDVFLHVWGYVFTWITLAIHYWAGPRITHILSRAGDILEIVMSAMRLEKFNSMTKDWLRLLEIPILAELAERIVEGDKRFGVIDKSGTPYELIKKTVEPENVPTALSKVESL,17.6058
+EFRTALGTLAAFVAIFFISVQFLFRFYPETWLPIYHLAFKRLSVPPAAIVAIASVTIFCIGAIFGLFPGPALALRRLTGNVAFLIGIATAIGIGVTFLIKGISNSNNTSSMSIIIRTVAGAIAVVLLTLPALVVRIHGNFGRAAVGGGAAEGANAAIFQSLTGSNNAFRDALFNFGVKLLFGLAIETREIILLEYIYNLLLTVGYDLNFASRGRLQLNAITGLLVVSAAIVSGYRTVAAERKVFDFALLKARKSVYPALRELRLVPLITFIGVLFVTTK,15.2478
+KGLQFTRKGWNHKGRRHWRDFDTVAGNALLGIEGQTGPRMVETGENVTTEPGNRRTRPTLLATGTEPADAGIEETRIEQDVILPLTTKANGGMIRVHHYDVRKIAGVEIDLESDILEARLTDGEDLHNCKFTTTVKAHIKTEPTPVSAADSEILLKGQYVSSDFEVLDSDLDANVSRDSRMWFEVAYICDILDKQTLLNEGMTFTVTSDGYSSGAADVWVLSTIKTQCRHAQGQQWLYRAGNLKPVVEMEIVYSAARDVTGGSLFGAVNSAAPFTVEMLFFPATIEQLRPGTPRAGKTITNPENATSGGNIEVFEEVKHFSDSQFRNEVRFITDDDSVYTATERRRIPNAPQNGIIRYWMKNGYASWNTEKVYARQPDGDITRQESFENAAQSMSTADNYYNHYYKEALRMHLAGGVEDDLEDDVTQEVRVSKDGEVEIDLDLNYTSKRYREGISWFLGCNAAHGIPINDAGVGFAFAAIPQYA,18.6838
+FSSVATATTTAIAFAIAAGVGGAIGGAVVGSLVIASLRGTVTAASALKAPLVPLALTVGAASLGATIGLAASWGVNLTL,10.8479
+SSITAAIQLYKPDSISILDDDSPDDLFETVEFLTEKQKNKQTSDNSYKLFADSFLSIVDSPNWTNMLLIAARVLLVLYTICPCCRADWVGAIGTDDVSYDIVCDLLGININFFKITKVLTAQYLPGRTKVGYMKHPLKTSYFVSIYVEDISDCARHPYGFSYAWQYVKKPYGTVSVDIYNGNPREKLFCLEGLNWATGLGLVVGAGAYKSLGTSVERVNTLVIFLETGELFVWAYAALWFRKRYTEDSEAKVNLYIAGLIVLFAVEKVSYAVPDIVFKEQILKSMVIFAKFSIINLYLDALFDFVFICIIILLLRVLSKDLEAVVGPTLSVFL,17.2645
+LECYGQQSSLIEMYRDYTIKVRDRYANNERIILDHYLVLNGDFYVRLASNKIVLGPDDANSVVAILQIGDMGLFLANGKNVTEMKRMLEKLEILYFTGSEAAVGSVTGHVCLMITNIWKDNKKLVEMLEFLGTEIIYNSVGLVFMIGKMSDKQGVYAKNKFSDSILEIAVKLQNFTWRNHVLFIGAYLYQWELYAEPEVVINNNISVIRVLWDPDGKSLYIIRPEKPPNIFEYLMHGICTFGGVGAIAGGMGVPASHIGGLIYKADFSISSWCEPGSVNVGALPYGSNCVVVQEGGNVVTFSLPTGSDVPIFALEHFPEPGKWKWEGFYWINPTDYRIMISGLKYTLAANAIAGIGAYLESYNIKISTWQYLVNGNPYDSVGVYNQHEYPLYPSLPMSDFTIFPVLTFAP,18.4071

benchmarks/MLM/config.py

@@ -0,0 +1,14 @@
+PATH = "/workspace/sg666/MDpLM"
+TRAIN_DATA = PATH + "/data/membrane/train.csv"
+TEST_DATA = PATH + "/data/membrane/test.csv"
+VAL_DATA = PATH + "/data/membrane/val.csv"
+
+ESM_MODEL_PATH = "facebook/esm2_t30_150M_UR50D"
+MLM_MODEL_PATH = PATH + "/benchmarks/MLM"
+CKPT_DIR = PATH + "/benchmarks/MLM/model_ckpts"
+
+ESM_LAYERS = 3
+BATCH_SIZE = 8
+NUM_EPOCHS = 10
+LEARNING_RATE = 5e-3
+MASKING_RATE = 0.40

benchmarks/MLM/data_loader.py

@@ -0,0 +1,48 @@
+import pandas as pd
+import torch
+import config
+import random
+from torch.utils.data import Dataset, DataLoader
+from torch.nn.utils.rnn import pad_sequence
+from pretrained_models import load_esm2_model
+
+class ProteinDataset(Dataset):
+    def __init__(self, csv_file, tokenizer):
+        self.tokenizer = tokenizer
+        self.data = pd.read_csv(csv_file)
+        self.max_len = max([len(seq) for seq in self.data['Sequence'].tolist()])
+
+    def __len__(self):
+        return len(self.data)
+
+    def __getitem__(self, idx):
+        sequence = self.data.iloc[idx]['Sequence'].upper()
+
+        # Randomly mask 15% of the sequence
+        num_masks = int(len(sequence) * 0.15)
+        mask_indices = random.sample(range(len(sequence)), num_masks)
+        masked_sequence = ''.join(["<mask>" if i in mask_indices else sequence[i] for i in range(len(sequence))])
+
+        inputs = self.tokenizer(masked_sequence, padding="max_length", truncation=True, max_length=self.max_len, return_tensors='pt')
+        input_ids = inputs['input_ids'].squeeze()
+        attention_mask = inputs['attention_mask'].squeeze()
+
+        labels = self.tokenizer(masked_sequence, return_tensors='pt', padding='max_length', max_length=self.max_len, truncation=True)['input_ids'].squeeze()
+        labels = torch.where(input_ids == self.tokenizer.mask_token_id, labels, -100)
+
+        return {"input_ids": input_ids, "attention_mask": attention_mask, "labels": labels}
+
+
+
+def get_dataloaders(config):
+    tokenizer, model = load_esm2_model(config.ESM_MODEL_PATH)
+    
+    train_dataset = ProteinDataset(config.TRAIN_DATA, tokenizer)
+    val_dataset = ProteinDataset(config.VAL_DATA, tokenizer)
+    test_dataset = ProteinDataset(config.TEST_DATA, tokenizer)
+    
+    train_loader = DataLoader(train_dataset, batch_size=config.BATCH_SIZE, shuffle=True)
+    val_loader = DataLoader(val_dataset, batch_size=config.BATCH_SIZE, shuffle=False)
+    test_loader = DataLoader(test_dataset, batch_size=config.BATCH_SIZE, shuffle=False)
+    
+    return train_loader, val_loader, test_loader

benchmarks/MLM/esm_utils.py

@@ -0,0 +1,16 @@
+import torch
+from transformers import AutoTokenizer, AutoModel, AutoModelForMaskedLM
+
+def load_esm2_model(model_name):
+    tokenizer = AutoTokenizer.from_pretrained(model_name)
+    masked_model = AutoModelForMaskedLM.from_pretrained(model_name)
+    embedding_model = AutoModel.from_pretrained(model_name)
+    return tokenizer, masked_model, embedding_model
+
+
+
+def get_latents(model, tokenizer, sequence):
+    inputs = tokenizer(sequence, return_tensors="pt").to(model.device)
+    with torch.no_grad():
+        outputs = model(**inputs)
+    return outputs.last_hidden_state.squeeze(0)

benchmarks/MLM/mlm_generate_utils.py

@@ -0,0 +1,108 @@
+import torch
+import math
+import config
+import sys
+import pandas as pd
+from esm_utils import get_latents
+from transformers import AutoModelForMaskedLM, AutoModel, AutoTokenizer
+
+
+def mask_for_de_novo(sequence_length):
+    return "<mask>" * sequence_length
+
+def generate_de_novo(sequence_length, tokenizer, model):
+    masked_sequence = mask_for_de_novo(sequence_length)
+    inputs = tokenizer(masked_sequence, return_tensors='pt').to(model.device)
+
+    with torch.no_grad():
+        logits = model(**inputs).logits
+    mask_token_indices = (inputs["input_ids"] == tokenizer.mask_token_id).nonzero(as_tuple=True)[1]
+    logits_at_masks = logits[0, mask_token_indices]
+
+    pred_tokens = []
+    for i in mask_token_indices:
+        topk_logits, topk_indices = logits_at_masks[i].topk(k=3, dim=-1)
+        probabilities = torch.nn.functional.softmax(topk_logits, dim=-1)
+        predicted_index = torch.distributions.categorical.Categorical(probabilities).sample()
+        predicted_token_id = topk_indices[predicted_index].item()
+        predicted_token = tokenizer.decode([predicted_token_id], skip_special_tokens=True)
+        pred_tokens.append(predicted_token)
+    
+    generated_sequence = ''.join(pred_tokens)
+    perplexity = calculate_perplexity(model, tokenizer, generated_sequence)
+
+    return (generated_sequence, perplexity)
+
+
+def mask_for_scaffold(sequence, generate_type):
+    if generate_type == "uppercase":
+        sequence = ''.join(["<mask>" if residue.isupper() else residue.upper() for residue in sequence])
+    elif generate_type == "lowercase":
+        sequence = ''.join(["<mask>" if residue.islower() else residue for residue in sequence])   
+    return sequence
+
+
+def generate_scaffold(sequence, generate_type, tokenizer, model):
+    masked_sequence = mask_for_scaffold(sequence, generate_type)
+    inputs = tokenizer(masked_sequence, return_tensors='pt').to(model.device)
+
+    with torch.no_grad():
+        logits = model(**inputs).logits
+    mask_token_indices = (inputs["input_ids"] == tokenizer.mask_token_id).nonzero(as_tuple=True)[1]
+    logits_at_masks = logits[0, mask_token_indices]
+
+    pred_tokens = []
+    for i in range(len(mask_token_indices)):
+        topk_logits, topk_indices = logits_at_masks[i].topk(k=3, dim=-1)
+        probabilities = torch.nn.functional.softmax(topk_logits, dim=-1)
+        predicted_index = torch.distributions.categorical.Categorical(probabilities).sample()
+        predicted_token_id = topk_indices[predicted_index].item()
+        predicted_token = tokenizer.decode([predicted_token_id], skip_special_tokens=True)
+
+        pred_tokens.append('G' if predicted_token == '' else predicted_token)
+
+    generated_sequence = masked_sequence
+    for token in pred_tokens:
+        generated_sequence = generated_sequence.replace("<mask>", token, 1)
+
+    return generated_sequence, mask_token_indices
+
+
+def calculate_perplexity(model, tokenizer, generated_sequence, mask_token_indices):
+    total_loss = 0.0
+    tensor_input = tokenizer.encode(generated_sequence, return_tensors='pt').to(model.device)
+
+    for i in mask_token_indices:
+        masked_input = tensor_input.clone()
+        masked_input[0, i] = tokenizer.mask_token_id
+    
+        labels = torch.full(tensor_input.shape, -100).to(model.device)
+        labels[0, i] = tensor_input[0, i]
+
+        with torch.no_grad():
+            outputs = model(masked_input, labels=labels)
+            total_loss += outputs.loss.item()
+    
+    num_mask_tokens = len(mask_token_indices)
+    if num_mask_tokens == 0:
+        perplexity = 10000
+    else:
+        avg_loss = total_loss / num_mask_tokens
+        perplexity = math.exp(avg_loss)
+
+    return perplexity
+
+
+def calculate_cosine_sim(original_sequence, generated_sequence, tokenizer, esm_model, device):
+    og_embeddings = get_latents(esm_model, tokenizer, original_sequence.upper()).to(device)
+    new_embeddings = get_latents(esm_model, tokenizer, generated_sequence).to(device)
+
+    sequence_similarity = torch.nn.functional.cosine_similarity(og_embeddings, new_embeddings, dim=-1)
+    cosine_similarity = torch.mean(sequence_similarity).item()
+    return cosine_similarity
+    
+
+def calculate_hamming_dist(original_sequence, generated_sequence):
+    generated_sequence = generated_sequence.upper()
+    original_sequence = original_sequence.upper()
+    return sum(1 if original_sequence[i] != generated_sequence[i] else 0 for i in range(len(original_sequence)))

benchmarks/MLM/mlm_motif_benchmarking.py

@@ -0,0 +1,39 @@
+import torch
+import config
+import sys
+import pandas as pd
+from mlm_generate_utils import generate_scaffold, calculate_perplexity, calculate_cosine_sim, calculate_hamming_dist
+from transformers import AutoModelForMaskedLM, AutoModel, AutoTokenizer
+
+def motif_benchmarking():
+    path = "/workspace/sg666/MDpLM"
+    
+    test_sequences = pd.read_csv(path + "/data/membrane/test.csv")['Sequence'].tolist()
+
+    tokenizer = AutoTokenizer.from_pretrained(config.CKPT_DIR + "/best_model_epoch")
+    mlm_model = AutoModelForMaskedLM.from_pretrained(config.CKPT_DIR + "/best_model_epoch")
+    esm_model = AutoModel.from_pretrained("facebook/esm2_t36_3B_UR50D")
+
+    device = torch.device('cuda' if torch.cuda.is_available() else "cpu")
+    mlm_model.to(device)
+    esm_model.to(device)
+
+    for generate_case in ['uppercase', 'lowercase']:
+        case_results = []
+        for original_sequence in test_sequences:
+            generated_sequence, mask_token_idx = generate_scaffold(original_sequence, generate_case, tokenizer, mlm_model)
+            perplexity = calculate_perplexity(mlm_model, tokenizer, generated_sequence, mask_token_idx)
+            cos_sim = calculate_cosine_sim(original_sequence, generated_sequence, tokenizer, esm_model, device)
+            hamming_distance = calculate_hamming_dist(original_sequence, generated_sequence)
+        
+            case_results.append([original_sequence, generated_sequence, perplexity, cos_sim, hamming_distance])
+
+            print(case_results)
+            sys.stdout.flush()
+
+        df = pd.DataFrame(case_results, columns=['Original Sequence', 'Generated Sequence', 'Perplexity', 'Cosine Similarity', 'Hamming Distance'])
+        df.to_csv(path + f'/benchmarks/MLM/mlm_{generate_case}_results.csv', index=False)
+
+
+if __name__ == "__main__":
+    motif_benchmarking()

benchmarks/MLM/model.py

@@ -0,0 +1,65 @@
+import config
+import torch
+import torch.nn as nn
+from pretrained_models import load_esm2_model
+from transformers import AutoModelForMaskedLM, AutoTokenizer, AutoModel
+
+class MembraneTokenizer:
+    def __init__(self, esm_model_path=config.ESM_MODEL_PATH):
+        self.tokenizer = AutoTokenizer.from_pretrained(esm_model_path)
+    
+    def __getattr__(self, name):
+        return getattr(self.tokenizer, name)
+
+    def __call__(self, *args, **kwargs):
+        return self.tokenizer(*args, **kwargs)
+    
+    def save_tokenizer(self, save_dir):
+        self.tokenizer.save_pretrained(save_dir)
+    
+    def load_tokenizer(self, load_dir):
+        self.tokenizer.save_pretrained(load_dir)
+
+class MembraneMLM:
+    def __init__(self, esm_model_path=config.ESM_MODEL_PATH):
+        self.model = AutoModelForMaskedLM.from_pretrained(esm_model_path)
+        self.tokenizer = AutoTokenizer.from_pretrained(esm_model_path)
+    
+    def __getattr__(self, name):
+        return getattr(self.model, name)
+
+    def __call__(self, *args, **kwargs):
+        return self.model(*args, **kwargs)
+
+    def freeze_model(self):
+       # Disable parameter updates for all layers
+        for param in self.model.parameters():
+            param.requires_grad = False
+
+    def unfreeze_n_layers(self):
+        # Count number of encoder layers
+        model_layers = len(self.model.esm.encoder.layer)
+
+        # Enable parameter updates for the last 3 encoder layers
+        for i, layer in enumerate(self.model.esm.encoder.layer):
+            if i >= model_layers-config.ESM_LAYERS:
+                for module in layer.attention.self.key.modules():
+                    for param in module.parameters():
+                        param.requires_grad = True
+                for module in layer.attention.self.query.modules():
+                    for param in module.parameters():
+                        param.requires_grad = True
+                for module in layer.attention.self.value.modules():
+                    for param in module.parameters():
+                        param.requires_grad = True
+    
+    def forward(self, **inputs):
+        return self.model(**inputs)
+
+    def save_model(self, save_dir):
+        self.model.save_pretrained(save_dir)
+        self.tokenizer.save_pretrained(save_dir)
+
+    def load_model(self, load_dir):
+        self.model = AutoModel.from_pretrained(load_dir)
+        self.tokenizer = AutoTokenizer.from_pretrained(load_dir)

benchmarks/MLM/pretrained_models.py

@@ -0,0 +1,12 @@
+import torch
+from transformers import AutoTokenizer, AutoModel, EsmForMaskedLM, AutoModelForMaskedLM
+
+def load_esm2_model(esm_model_path):
+    tokenizer = AutoTokenizer.from_pretrained(esm_model_path)
+    model = AutoModelForMaskedLM.from_pretrained(esm_model_path)
+    return tokenizer, model
+
+def load_mlm_model(esm_model_path, ckpt_path):
+    tokenizer = AutoTokenizer.from_pretrained(esm_model_path)
+    model = AutoModelForMaskedLM.from_pretrained(ckpt_path)
+    return tokenizer, model

benchmarks/MLM/screen_mlm_cosine_hamming.py

@@ -0,0 +1,17 @@
+import pandas as pd
+
+path = "/home/sg666/MDpLM/benchmarks/MLM"
+
+df = pd.read_csv(path + "/mlm_uppercase_results.csv")
+
+all_sequences = df['Original Sequence'].tolist()
+seq_len_sum = sum(len(seq) for seq in all_sequences)
+ppls = [ppl for ppl in df['Perplexity'].tolist() if ppl != 10000]
+
+ppl_mean = sum(ppls) / len(ppls)
+cos_mean = df.loc[:, 'Cosine Similarity'].mean()
+hamming_mean = sum(dist for dist in df['Hamming Distance'].tolist()) / seq_len_sum
+
+print(ppl_mean)
+print(cos_mean)
+print(hamming_mean)

benchmarks/MLM/train_and_test.py

@@ -0,0 +1,184 @@
+import torch
+import config
+import math
+import sys
+import os
+from tqdm import tqdm
+from torch.optim import Adam
+from torch.optim.lr_scheduler import CosineAnnealingLR
+from transformers import AutoModelForMaskedLM, AutoModel, AutoTokenizer, AutoConfig
+from pretrained_models import load_esm2_model
+from model import MembraneMLM, MembraneTokenizer
+from data_loader import get_dataloaders
+
+def save_hyperparams(ckpt_dir):
+    hyperparms_txt_file = os.path.join(ckpt_dir, "hyperparameters.txt")
+    with open(hyperparms_txt_file, 'w') as f:
+        for k, v in vars(config).items():
+            if k.isupper():
+                f.write(f"{k}: {v}\n")
+
+def train_and_validate(model, optimizer, device, train_loader, val_loader, num_epochs, ckpt_dir):
+    best_val_loss = float('inf')
+
+    for epoch in range(num_epochs):
+        print(f"EPOCH {epoch+1}/{num_epochs}")
+        sys.stderr.flush()
+        total_train_loss = 0.0
+        weighted_total_train_loss = 0.0
+        total_masked_train_tokens = 0
+        
+        model.train()
+        train_update_interval = len(train_loader) // 4
+
+        with tqdm(enumerate(train_loader), desc="Training batch", total=len(train_loader), leave=True, position=0, ncols=100) as trainbar:
+            for step, inputs in trainbar:
+                inputs = {k: v.to(device) for k, v in inputs.items()}
+                optimizer.zero_grad()
+                outputs = model(**inputs)
+                train_loss = outputs.loss
+                train_loss.backward()
+                optimizer.step()
+
+                num_mask_tokens = (inputs["input_ids"] == tokenizer.mask_token_id).sum().item()
+                total_masked_train_tokens += num_mask_tokens
+
+                total_train_loss += train_loss.item()
+                weighted_total_train_loss += train_loss.item() * num_mask_tokens
+
+                if (step+1) % train_update_interval == 0:
+                    trainbar.update(train_update_interval)
+
+            avg_train_loss = total_train_loss / len(train_loader)
+            avg_train_neg_log_likelihood = weighted_total_train_loss / total_masked_train_tokens
+            train_perplexity = math.exp(avg_train_neg_log_likelihood)
+
+        # Save model every epoch
+        train_ckpt_path = os.path.join(config.CKPT_DIR, f'epoch{epoch+1}')
+        model.save_model(train_ckpt_path)
+        save_hyperparams(train_ckpt_path)
+
+        # Validate model
+        if val_loader:
+            model.eval()
+            total_val_loss = 0.0
+            weighted_total_val_loss = 0.0
+            total_masked_val_tokens = 0.0
+
+            with torch.no_grad():
+                val_update_interval = len(val_loader) // 4
+
+                with tqdm(enumerate(val_loader), desc='Validiation batch', total=len(val_loader), leave=True, position=0) as valbar:
+                    for step, inputs in valbar:
+                        inputs = {k: v.to(device) for k, v in inputs.items()}
+                        val_loss = model(**inputs).loss.item()
+
+                        num_mask_tokens = (inputs['input_ids'] == tokenizer.mask_token_id).sum().item()
+                        total_masked_val_tokens += num_mask_tokens
+                        
+                        total_val_loss += val_loss
+                        weighted_total_val_loss += val_loss * num_mask_tokens
+
+                        if (step+1) % val_update_interval == 0:
+                            valbar.update(val_update_interval)
+
+                avg_val_loss = total_val_loss / len(val_loader)
+                avg_val_neg_log_likelihood = weighted_total_val_loss / total_masked_val_tokens
+                val_perplexity = math.exp(avg_val_neg_log_likelihood)
+
+        # Save the best model based on validation loss
+        if avg_val_loss < best_val_loss:
+            best_val_loss = avg_val_loss
+            val_ckpt_path = os.path.join(config.CKPT_DIR, "best_model_epoch")
+            model.save_model(val_ckpt_path)
+            save_hyperparams(val_ckpt_path)
+
+
+        print(f"Average train loss: {avg_train_loss}")
+        print(f"Average train perplexity: {train_perplexity}\n")
+        sys.stdout.flush()
+
+        print(f"Average validation loss: {avg_val_loss}")
+        print(f"Average validation perplexity: {val_perplexity}\n")
+        sys.stdout.flush()
+        
+
+    return avg_train_loss, train_perplexity, avg_val_loss, val_perplexity
+                            
+
+def test(model, test_loader, device):
+    model.to(device).eval()
+    total_test_loss = 0.0
+    weighted_total_test_loss = 0.0
+    total_masked_test_tokens = 0.0
+
+    with torch.no_grad():
+        for step, inputs in enumerate(test_loader):
+            inputs = {k: v.to(device) for k, v in inputs.items()}
+            outputs = model(**inputs)
+            test_loss = outputs.loss.item()
+
+            num_mask_tokens = (inputs["input_ids"] == tokenizer.mask_token_id).sum().item()
+            total_masked_test_tokens += num_mask_tokens
+
+            total_test_loss += test_loss
+            weighted_total_test_loss += test_loss * num_mask_tokens
+        
+        avg_test_loss = total_test_loss / len(test_loader)
+        avg_test_neg_log_likilehood = weighted_total_test_loss / total_masked_test_tokens
+        test_perplexity = math.exp(avg_test_neg_log_likilehood)
+
+    return avg_test_loss, test_perplexity
+
+
+if __name__ == "__main__":
+    device = torch.device('cuda' if torch.cuda.is_available() else "cpu")
+    print(device)
+    
+    model = MembraneMLM()
+    model.to(device)
+    model.freeze_model()
+    model.unfreeze_n_layers()
+    tokenizer = model.tokenizer
+
+    train_loader, val_loader, test_loader = get_dataloaders(config)
+    optimizer = Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=config.LEARNING_RATE)
+    
+    # Train and test the model
+    avg_train_loss, train_ppl, avg_val_loss, val_ppl = train_and_validate(model, optimizer, device, train_loader, val_loader, config.NUM_EPOCHS, config.CKPT_DIR)
+    avg_test_loss, test_ppl = test(model, test_loader, device)
+
+    results_dict = {"Average train loss": avg_train_loss,
+                    "Average train perplexity": train_ppl,
+                    "Average val loss": avg_val_loss,
+                    "Average val perplexity": val_ppl,
+                    "Average test loss": avg_test_loss,
+                    "Average test perplexity": test_ppl,
+    }
+
+    print("TRAIN AND TEST RESULTS")
+    for k, v in results_dict.items():
+        print(f"{k}: {v}\n")
+
+    # Save training and test performance
+    with open(config.CKPT_DIR + "/train_test_results.txt", 'w') as f:
+        for k, v in results_dict.items():
+            f.write(f'{k}: {v}\n')
+
+
+### Get embeddings from model
+    # best_model_pth = config.MLM_MODEL_PATH + "/best_model"
+
+    # model = AutoModel.from_pretrained(best_model_pth)
+    # tokenizer = AutoTokenizer.from_pretrained(best_model_pth)
+    # model.eval().to(device)
+
+    # random_seq = "WPIQMVYSLGQHADYMQWFTIMPPPIEMIFVWHNCTQHDYSFRERAGEVDQARMKTEMAR"
+    # inputs = tokenizer(random_seq, return_tensors='pt')
+    # inputs = {k: v.to(device) for k, v in inputs.items()}
+    # inputs = inputs['input_ids']
+    # print(inputs)
+    # with torch.no_grad():
+    #     outputs = model(inputs).last_hidden_state
+    #     print(outputs)
+    #     print(outputs.size())

benchmarks/Supervised/Localization/cell_localization_predictor.py

@@ -0,0 +1,224 @@
+import torch
+import torch.nn as nn
+import torch.optim as optim
+from torch.utils.data import DataLoader, Dataset
+from transformers import AutoModel, AutoTokenizer
+from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, roc_auc_score
+
+from tqdm import tqdm
+from datetime import datetime
+import pandas as pd
+import numpy as np
+import pickle
+import os
+
+# Hyperparameters dictionary
+path = "/workspace/sg666/MDpLM"
+
+hyperparams = {
+    "batch_size": 1,
+    "learning_rate": 5e-4,
+    "num_epochs": 5,
+    "esm_model_path": "facebook/esm2_t33_650M_UR50D",
+    'mlm_model_path': path + "/benchmarks/MLM/model_ckpts/best_model_epoch",
+    "mdlm_model_path": path + "/checkpoints/membrane_automodel/epochs30_lr3e-4_bsz16_gradclip1_beta-one0.9_beta-two0.999_bf16_all-params",
+    "train_data": path + "/benchmarks/Supervised/Localization/true_deeploc2.0_cell-local_train-val.csv",
+    "test_data" : path + "/benchmarks/Supervised/Localization/true_deeploc2.0_cell-local_test.csv",
+}
+
+# Helper functions to obtain all embeddings for a sequence
+def load_models(esm_model_path, mlm_model_path, mdlm_model_path):
+    esm_tokenizer = AutoTokenizer.from_pretrained(esm_model_path)
+    esm_model = AutoModel.from_pretrained(esm_model_path).to(device)
+    mlm_model = AutoModel.from_pretrained(mlm_model_path).to(device)
+    mdlm_model = AutoModel.from_pretrained(mdlm_model_path).to(device)
+
+    return esm_tokenizer, esm_model, mlm_model, mdlm_model
+
+def get_latents(embedding_type, tokenizer, esm_model, mlm_model, mdlm_model, sequence, device):
+    if embedding_type == "esm":
+        inputs = tokenizer(sequence, return_tensors='pt').to(device)
+        with torch.no_grad():
+            embeddings = esm_model(**inputs).last_hidden_state.squeeze(0)
+
+    elif embedding_type == "mlm":
+        inputs = tokenizer(sequence, return_tensors='pt')['input_ids'].to(device)
+        with torch.no_grad():
+            embeddings = mlm_model(inputs).last_hidden_state.squeeze(0)
+
+    elif embedding_type == "mdlm":
+        inputs = tokenizer(sequence, return_tensors='pt')['input_ids'].to(device)
+        with torch.no_grad():
+            embeddings = mdlm_model(inputs).last_hidden_state.squeeze(0)
+    
+    return embeddings
+
+
+# Dataset class can load pickle file
+class LocalizationDataset(Dataset):
+    def __init__(self, embedding_type, csv_file, esm_model_path, mlm_model_path, mdlm_model_path, device):
+        self.data = pd.read_csv(csv_file)
+        self.data = self.data[self.data['Sequence'].apply(len) < 1024].reset_index(drop=True)
+        self.embedding_type = embedding_type
+        self.tokenizer, self.esm_model, self.mlm_model, self.mdlm_model = load_models(esm_model_path, mlm_model_path, mdlm_model_path)
+        self.device = device
+
+    def __len__(self):
+        return len(self.data)
+
+    def __getitem__(self, idx):
+        sequence = self.data.iloc[idx]['Sequence']
+        embeddings = get_latents(self.embedding_type, self.tokenizer, self.mlm_model, self.esm_model, self.mdlm_model,
+                                 sequence, self.device)
+
+        label = 0 if self.data.iloc[idx]['Cell membrane'] == 0 else 1
+        labels = torch.tensor(label, dtype=torch.float32).view(1,1).squeeze(-1)
+
+        return embeddings, labels
+
+# Predict localization with MLP head using pooled embeddings
+class LocalizationPredictor(nn.Module):
+    def __init__(self, input_dim):
+        super(LocalizationPredictor, self).__init__()
+        self.classifier = nn.Sequential(
+            nn.Linear(input_dim, 640),
+            nn.ReLU(),
+            nn.Linear(640, 1)
+        )
+
+    def forward(self, embeddings):
+        logits = self.classifier(embeddings)
+        logits = torch.mean(logits, dim=1)
+        probs = torch.nn.functional.softmax(logits)
+        return probs
+
+# Training function
+def train(model, dataloader, optimizer, criterion, device):
+    model.train()
+    total_loss = 0
+    for embeddings, labels in tqdm(dataloader):
+        embeddings, labels = embeddings.to(device), labels.to(device)
+        optimizer.zero_grad()
+        outputs = model(embeddings)
+        loss = criterion(outputs, labels)
+        loss.backward()
+        optimizer.step()
+        total_loss += loss.item()
+    return total_loss / len(dataloader)
+
+# Evaluation function
+def evaluate(model, dataloader, device):
+    model.eval()
+    preds, true_labels = [], []
+    with torch.no_grad():
+        for embeddings, labels in tqdm(dataloader):
+            embeddings, labels = embeddings.to(device), labels.to(device)
+            outputs = model(embeddings)
+            preds.append(outputs.cpu().numpy())
+            true_labels.append(labels.cpu().numpy())
+    return preds, true_labels
+
+# Metrics calculation
+def calculate_metrics(preds, labels, threshold=0.5):
+    all_metrics = []
+    for pred, label in zip(preds, labels):
+        pred = (pred > threshold).astype(int)
+
+        accuracy = accuracy_score(label, pred)
+        precision = precision_score(label, pred, average='macro')
+        recall = recall_score(label, pred, average='macro')
+        f1_macro = f1_score(label, pred, average='macro')
+        f1_micro = f1_score(label, pred, average='micro')
+        
+        all_metrics.append([accuracy, precision, recall, f1_macro, f1_micro])
+    
+    avg_metrics = np.mean(all_metrics, axis=0)
+    print(avg_metrics)
+    return avg_metrics
+
+
+
+if __name__ == "__main__":
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+    for embedding_type in ['mdlm', 'esm', 'mlm']:
+        # Initialize datasets
+        train_dataset = LocalizationDataset(embedding_type,
+                                            hyperparams['train_data'],
+                                            hyperparams['esm_model_path'],
+                                            hyperparams['mlm_model_path'],
+                                            hyperparams['mdlm_model_path'],
+                                            device)
+        test_dataset = LocalizationDataset(embedding_type,
+                                           hyperparams['test_data'],
+                                           hyperparams['esm_model_path'],
+                                           hyperparams['mlm_model_path'],
+                                           hyperparams['mdlm_model_path'],
+                                           device)
+
+        # Prepare dataloaders
+        train_dataloader = DataLoader(train_dataset, batch_size=hyperparams["batch_size"], shuffle=True)
+        test_dataloader = DataLoader(test_dataset, batch_size=hyperparams["batch_size"], shuffle=False)
+
+        # Initialize model, optimizer, and loss function
+        input_dim=640 if embedding_type=="mdlm" else 1280
+        model = LocalizationPredictor(input_dim=input_dim).to(device)
+        optimizer = optim.Adam(model.parameters(), lr=hyperparams["learning_rate"])
+        criterion = nn.BCELoss()
+
+        # Initialize main directory model checkpoints
+        base_checkpoint_dir = f"{path}/benchmarks/Supervised/Localization/model_checkpoints/{embedding_type}"
+        # Initialize subdirectory and name it based on hyperparameters
+        hyperparam_str = f"batch_{hyperparams['batch_size']}_lr_{hyperparams['learning_rate']}_epochs_{hyperparams['num_epochs']}"
+        model_checkpoint_dir = os.path.join(base_checkpoint_dir, hyperparam_str)
+        os.makedirs(model_checkpoint_dir, exist_ok=True)
+
+
+        # Training loop
+        for epoch in range(hyperparams["num_epochs"]):
+            # Train the model
+            train_loss = train(model, train_dataloader, optimizer, criterion, device)
+            print(f"EPOCH {epoch+1}/{hyperparams['num_epochs']}")
+            print(f"TRAIN LOSS: {train_loss:.4f}")
+            print("\n")
+
+            # Save the model checkpoint for the current epoch
+            checkpoint_path = os.path.join(model_checkpoint_dir, f"epoch{epoch + 1}.pth")
+            torch.save({
+                'epoch': epoch + 1,
+                'model_state_dict': model.state_dict(),
+                'optimizer_state_dict': optimizer.state_dict(),
+                'loss': train_loss,
+            }, checkpoint_path)
+            print(f"Checkpoint saved at {checkpoint_path}\n")
+
+            # Save hyperparameters only once
+            if epoch == 0:  # Hyperparameters don't change midway through training
+                hyperparams_file = os.path.join(model_checkpoint_dir, "hyperparams.txt")
+                with open(hyperparams_file, 'w') as f:
+                    for key, value in hyperparams.items():
+                        f.write(f"{key}: {value}\n")
+                print(f"Hyperparameters saved at {hyperparams_file}\n")
+
+        # Evaluate model on test dataset
+        print("Test set")
+        test_preds, test_labels = evaluate(model, test_dataloader, device)
+        test_metrics = calculate_metrics(test_preds, test_labels)
+        print(test_metrics)
+        print("TEST METRICS:")
+        print(f"Accuracy: {test_metrics[0]:.4f}")
+        print(f"Precision: {test_metrics[1]:.4f}")
+        print(f"Recall: {test_metrics[2]:.4f}")
+        print(f"F1 Macro Score: {test_metrics[3]:.4f}")
+        print(f"F1 Micro Score: {test_metrics[4]:.4f}")
+
+        #Save test results
+        test_results_file = os.path.join(model_checkpoint_dir, "test_results.txt")
+        with open(test_results_file, 'w') as f:
+            f.write("TEST METRICS:\n")
+            f.write(f"Accuracy: {test_metrics[0]:.4f}\n")
+            f.write(f"Precision: {test_metrics[1]:.4f}\n")
+            f.write(f"Recall: {test_metrics[2]:.4f}\n")
+            f.write(f"F1 Macro Score: {test_metrics[3]:.4f}\n")
+            f.write(f"F1 Micro: {test_metrics[4]:.4f}\n")
+        print(f"Test results saved at {test_results_file}\n")

benchmarks/Supervised/Localization/process_cell_local_data.py

@@ -0,0 +1,12 @@
+import pandas as pd
+
+path = "/home/sg666/MDpLM/benchmarks/Supervised/Localization"
+
+train_val = pd.read_csv(path + "/deeploc2.0_train_val.csv")
+test = pd.read_csv(path + "/deeploc2.0_test.csv")
+
+train_val = train_val[train_val['Sequence'].apply(len) < 1024].reset_index(drop=True)
+test = test[test['Sequence'].apply(len) < 1024].reset_index(drop=True)
+
+train_val.to_csv(path + "/true_deeploc2.0_cell-local_train-val.csv", index=False)
+test.to_csv(path + "/true_deeploc2.0_cell-local_test.csv", index=False)

benchmarks/Supervised/Localization/true_deeploc2.0_cell-local_train-val.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0ca38d78cc8fbc8777a23f456477901f5af4bbfda7a0908081effd09adbe7e94
+size 12568908

benchmarks/Supervised/Membrane Type/membrane_type_predictor.py

@@ -0,0 +1,226 @@
+import torch
+import torch.nn as nn
+import torch.optim as optim
+from torch.utils.data import DataLoader, Dataset
+from transformers import AutoModel, AutoTokenizer
+from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, roc_auc_score
+
+from tqdm import tqdm
+from datetime import datetime
+import pandas as pd
+import numpy as np
+import pickle
+import os
+
+# Hyperparameters dictionary
+path = "/workspace/sg666/MDpLM"
+
+hyperparams = {
+    "batch_size": 1,
+    "learning_rate": 5e-4,
+    "num_epochs": 5,
+    "esm_model_path": "facebook/esm2_t33_650M_UR50D",
+    'mlm_model_path': path + "/benchmarks/MLM/model_ckpts/best_model_epoch",
+    "mdlm_model_path": path + "/checkpoints/membrane_automodel/epochs30_lr3e-4_bsz16_gradclip1_beta-one0.9_beta-two0.999_bf16_all-params",
+    "train_data": path + "/benchmarks/Supervised/Membrane Type/membrane_type_train.csv",
+    "test_data" : path + "/benchmarks/Supervised/Membrane Type/membrane_type_test.csv",
+}
+
+# Helper functions to obtain all embeddings for a sequence
+def load_models(esm_model_path, mlm_model_path, mdlm_model_path):
+    esm_tokenizer = AutoTokenizer.from_pretrained(esm_model_path)
+    esm_model = AutoModel.from_pretrained(esm_model_path).to(device)
+    mlm_model = AutoModel.from_pretrained(mlm_model_path).to(device)
+    mdlm_model = AutoModel.from_pretrained(mdlm_model_path).to(device)
+    return esm_tokenizer, esm_model, mlm_model, mdlm_model
+
+def get_latents(embedding_type, tokenizer, esm_model, mlm_model, mdlm_model, sequence, device):
+    if embedding_type == "esm":
+        inputs = tokenizer(sequence, return_tensors='pt').to(device)
+        with torch.no_grad():
+            outputs = esm_model(**inputs)
+            embeddings = outputs.last_hidden_state.squeeze(0)
+
+    elif embedding_type == "mlm":
+        inputs = tokenizer(sequence, return_tensors='pt').to(device)
+        with torch.no_grad():
+            embeddings = mlm_model(**inputs).last_hidden_state.squeeze(0)
+
+    elif embedding_type == "mdlm":
+        inputs = tokenizer(sequence, return_tensors="pt").to(device)
+        with torch.no_grad():
+            embeddings = mdlm_model(**inputs).last_hidden_state.squeeze(0)
+    
+    return embeddings
+
+
+# Dataset class can load pickle file
+class MembraneDataset(Dataset):
+    def __init__(self, embedding_type, csv_file, esm_model_path, mlm_model_path, mdlm_model_path, device):
+        self.data = pd.read_csv(csv_file)
+        self.data = self.data[self.data['Sequence'].apply(len) < 1024].reset_index(drop=True)
+
+        self.embedding_type = embedding_type
+        self.device = device
+
+        self.tokenizer, self.esm_model, self.mlm_model, self.mdlm_model = load_models(esm_model_path, mlm_model_path, mdlm_model_path)
+        
+        # Create multi-class label list
+        self.data['label'] = self.data.iloc[:, 3:7].values.tolist()
+        self.data['label'] = self.data['label']
+
+    def __len__(self):
+        return len(self.data)
+
+    def __getitem__(self, idx):
+        sequence = self.data.iloc[idx]['Sequence']
+        embeddings = get_latents(self.embedding_type, self.tokenizer, self.esm_model, self.mlm_model, self.mdlm_model,
+                                 sequence, self.device)
+        labels = torch.tensor(self.data.iloc[idx]['label'], dtype=torch.float32)
+
+        return embeddings, labels
+
+
+# Predict localization with MLP head using pooled embeddings
+class MembranePredictor(nn.Module):
+    def __init__(self, input_dim, num_classes):
+        super(MembranePredictor, self).__init__()
+        self.classifier = nn.Sequential(
+            nn.Linear(input_dim, 640),
+            nn.ReLU(),
+            nn.Linear(640, num_classes)
+        )
+
+    def forward(self, embeddings):
+        logits = self.classifier(embeddings)
+        logits = torch.mean(logits, dim=1)
+        probs = torch.sigmoid(logits)
+        return probs # pass logits of dimension 1x8 (8-class distribution) to CE loss
+
+# Training function
+def train(model, dataloader, optimizer, criterion, device):
+    model.train()
+    total_loss = 0
+    for embeddings, labels in tqdm(dataloader):
+        embeddings, labels = embeddings.to(device), labels.to(device)
+        optimizer.zero_grad()
+        outputs = model(embeddings)
+        loss = criterion(outputs, labels)
+        loss.backward()
+        optimizer.step()
+        total_loss += loss.item()
+    return total_loss / len(dataloader)
+
+# Evaluation function
+def evaluate(model, dataloader, device):
+    model.eval()
+    preds, true_labels = [], []
+    with torch.no_grad():
+        for embeddings, labels in tqdm(dataloader):
+            embeddings, labels = embeddings.to(device), labels.to(device)
+            outputs = model(embeddings)
+            preds.append(outputs.cpu().numpy())
+            true_labels.append(labels.cpu().numpy())
+    return preds, true_labels
+
+# Metrics calculation
+def calculate_metrics(preds, labels, threshold=0.5):
+    all_metrics = []
+    for pred, label in zip(preds, labels):
+        pred = (pred > threshold).astype(int)
+
+        accuracy = accuracy_score(label, pred)
+        precision = precision_score(label, pred, average='macro')
+        recall = recall_score(label, pred, average='macro')
+        f1_macro = f1_score(label, pred, average='macro')
+        f1_micro = f1_score(label, pred, average='micro')
+        
+        all_metrics.append([accuracy, precision, recall, f1_macro, f1_micro])
+    
+    avg_metrics = np.mean(all_metrics, axis=0)
+    return avg_metrics
+
+
+if __name__ == "__main__":
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+    for embedding_type in ['mdlm', 'mlm', 'esm']:
+        # Initialize datasets
+        train_dataset = MembraneDataset(embedding_type,
+                                        hyperparams['train_data'],
+                                        hyperparams['esm_model_path'],
+                                        hyperparams['mlm_model_path'],
+                                        hyperparams['mdlm_model_path'],
+                                        device)
+        test_dataset = MembraneDataset(embedding_type,
+                                       hyperparams['test_data'],
+                                       hyperparams['esm_model_path'],
+                                       hyperparams['mlm_model_path'],
+                                       hyperparams['mdlm_model_path'],
+                                       device)
+
+        # Prepare dataloaders
+        train_dataloader = DataLoader(train_dataset, batch_size=hyperparams["batch_size"], shuffle=True)
+        test_dataloader = DataLoader(test_dataset, batch_size=hyperparams["batch_size"], shuffle=False)
+
+        # Initialize model, optimizer, and loss function
+        input_dim=640 if embedding_type=="mdlm" else 1280
+        model = MembranePredictor(input_dim=input_dim, num_classes=4).to(device)
+        optimizer = optim.Adam(model.parameters(), lr=hyperparams["learning_rate"])
+        criterion = nn.CrossEntropyLoss()
+
+        # Initialize main directory model checkpoints
+        base_checkpoint_dir = f"{path}/benchmarks/Supervised/Membrane Type/model_checkpoints/{embedding_type}"
+        # Initialize subdirectory and name it based on hyperparameters
+        hyperparam_str = f"batch_{hyperparams['batch_size']}_lr_{hyperparams['learning_rate']}_epochs_{hyperparams['num_epochs']}"
+        model_checkpoint_dir = os.path.join(base_checkpoint_dir, hyperparam_str)
+        os.makedirs(model_checkpoint_dir, exist_ok=True)
+
+        # Training loop
+        for epoch in range(hyperparams["num_epochs"]):
+            # Train the model
+            train_loss = train(model, train_dataloader, optimizer, criterion, device)
+            print(f"EPOCH {epoch+1}/{hyperparams['num_epochs']}")
+            print(f"TRAIN LOSS: {train_loss:.4f}")
+            print("\n")
+
+            # Save the model checkpoint for the current epoch
+            checkpoint_path = os.path.join(model_checkpoint_dir, f"epoch{epoch + 1}.pth")
+            torch.save({
+                'epoch': epoch + 1,
+                'model_state_dict': model.state_dict(),
+                'optimizer_state_dict': optimizer.state_dict(),
+                'loss': train_loss,
+            }, checkpoint_path)
+            print(f"Checkpoint saved at {checkpoint_path}\n")
+
+            # Save hyperparameters only once
+            if epoch == 0:  # Hyperparameters don't change midway through training
+                hyperparams_file = os.path.join(model_checkpoint_dir, "hyperparams.txt")
+                with open(hyperparams_file, 'w') as f:
+                    for key, value in hyperparams.items():
+                        f.write(f"{key}: {value}\n")
+                print(f"Hyperparameters saved at {hyperparams_file}\n")
+
+
+        # Evaluate model on test dataset
+        print("Test set")
+        test_preds, test_labels = evaluate(model, test_dataloader, device)
+        test_metrics = calculate_metrics(test_preds, test_labels)
+        print("TEST METRICS:")
+        print(f"Accuracy: {test_metrics[0]:.4f}")
+        print(f"Precision: {test_metrics[1]:.4f}")
+        print(f"Recall: {test_metrics[2]:.4f}")
+        print(f"F1 Macro Score: {test_metrics[3]:.4f}")
+        print(f"F1 Micro Score: {test_metrics[4]:.4f}")
+
+        # Save test results
+        test_results_file = os.path.join(model_checkpoint_dir, "test_results.txt")
+        with open(test_results_file, 'w') as f:
+            f.write("TEST METRICS:\n")
+            f.write(f"Accuracy: {test_metrics[0]:.4f}\n")
+            f.write(f"Precision: {test_metrics[1]:.4f}\n")
+            f.write(f"Recall: {test_metrics[2]:.4f}\n")
+            f.write(f"F1 Macro Score: {test_metrics[3]:.4f}\n")
+            f.write(f"F1 Micro: {test_metrics[4]:.4f}\n")
+        print(f"Test results saved at {test_results_file}\n")

benchmarks/Supervised/Membrane Type/membrane_type_train.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:16b8eec677afa2de578d04ee1a0fc9582b2f8cfc47622cbd6374309cd6ab96f3
+size 12335695

benchmarks/Supervised/Membrane Type/split_membrane_type_data.py

@@ -0,0 +1,15 @@
+# Splits the DeepLoc 2.1 membrane type data into train/val and testing splits
+# Partition value of "4" indicates testing data
+
+import pandas as pd
+
+path = "/workspace/a03-sgoel/MDpLM/benchmarks/DeepLoc/Membrane Type"
+
+df = pd.read_csv(path + "/unsplit_membrane_type_all.csv")
+df = df.drop(columns=['Unnamed: 0'])
+
+train = df[df['Partition'] != 4]
+test = df[df['Partition'] == 4]
+
+train.to_csv(path + "/membrane_type_train.csv", index=False)
+test.to_csv(path + "/membrane_type_test.csv", index=False)

benchmarks/Supervised/Membrane Type/unsplit_membrane_type_all.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d878da32a06092f880262048e3c1eb692721c274b0a458fcc712a0dcbd80c71
+size 15683507

benchmarks/Supervised/Solubility/solubility_transformer.py

@@ -0,0 +1,353 @@
+import torch
+import torch.nn as nn
+import torch.optim as optim
+from torch.utils.data import DataLoader, Dataset
+from transformers import AutoModel, AutoTokenizer
+from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, roc_auc_score
+from sklearn.model_selection import ParameterGrid
+from tqdm import tqdm
+import pandas as pd
+import numpy as np
+import sys
+import os
+from datetime import datetime
+import logging
+
+logging.getLogger("transformers").setLevel(logging.ERROR)
+
+# Hyperparameters dictionary
+path = "/workspace/sg666/MDpLM"
+hyperparams = {
+    "train_data": path + "/data/membrane/train.csv",
+    "val_data": path + "/data/membrane/val.csv",
+    "test_data": path + "/data/membrane/test.csv",
+    'esm_model_path': "facebook/esm2_t33_650M_UR50D",
+    'mlm_model_path': path + "/benchmarks/MLM/model_ckpts/best_model_epoch",
+    "mdlm_model_path": path + "/checkpoints/membrane_automodel/epochs30_lr3e-4_bsz16_gradclip1_beta-one0.9_beta-two0.999_bf16_all-params",
+    "batch_size": 1,
+    "learning_rate": 5e-5,
+    "num_epochs": 2,
+    "num_layers": 4,
+    "num_heads": 16,
+    "dropout": 0.5
+}
+
+
+# Helper functions to obtain all embeddings for a sequence
+def load_models(esm_model_path, mlm_model_path, mdlm_model_path):
+    esm_tokenizer = AutoTokenizer.from_pretrained(esm_model_path)
+    esm_model = AutoModel.from_pretrained(esm_model_path).to(device)
+    mlm_model = AutoModel.from_pretrained(mlm_model_path).to(device)
+    mdlm_model = AutoModel.from_pretrained(mdlm_model_path).to(device)
+    return esm_tokenizer, esm_model, mlm_model, mdlm_model
+
+
+def get_latents(embedding_type, esm_model_path, mlm_model_path, mdlm_model_path, sequence, device):
+    tokenizer, esm_model, mlm_model, mdlm_model = load_models(esm_model_path, mlm_model_path, mdlm_model_path)
+
+    if embedding_type == "esm":
+        model = esm_model
+    elif embedding_type == "mlm":
+        model = mlm_model
+    elif embedding_type == "mdlm":
+        model = mdlm_model
+
+    inputs = tokenizer(sequence.upper(), return_tensors="pt").to(device)['input_ids']
+    with torch.no_grad():
+        embeddings = model(inputs).last_hidden_state.squeeze(0)[1:-1]
+    
+    return embeddings
+
+
+# Dataset class that loads embeddings and labels
+class SolubilityDataset(Dataset):
+    def __init__(self, embedding_type, csv_file, esm_model_path, mlm_model_path, mdlm_model_path, device):
+        self.data = pd.read_csv(csv_file).head(5)
+        #self.data = self.data[self.data['Sequence'].apply(len) < 1024].reset_index(drop=True)
+        self.embedding_type = embedding_type
+        self.esm_model_path = esm_model_path
+        self.mlm_model_path = mlm_model_path
+        self.mdlm_model_path = mdlm_model_path
+        self.device = device
+
+    def __len__(self):
+        return len(self.data)
+
+    def __getitem__(self, idx):
+        sequence = self.data.iloc[idx]['Sequence']
+        seq_len = len(sequence)
+        embeddings = get_latents(self.embedding_type, self.esm_model_path, self.mlm_model_path, self.mdlm_model_path,
+                                 sequence, self.device)
+        # Lowercase residues = soluble, uppercase = insoluble
+        label = [0 if residue.islower() else 1 for residue in sequence]
+        labels = torch.tensor(label, dtype=torch.float32)
+
+        return embeddings, labels, seq_len
+
+# Transformer model class
+class SolubilityPredictor(nn.Module):
+    def __init__(self, input_dim, hidden_dim, num_heads, num_layers, dropout):
+        super(SolubilityPredictor, self).__init__()
+        #self.embedding_dim = input_dim
+        # self.self_attention = nn.MultiheadAttention(input_dim, num_heads, dropout)
+        # encoder_layer = nn.TransformerEncoderLayer(
+        #     d_model=hidden_dim,
+        #     nhead=num_heads,
+        #     dropout=dropout,
+        #     batch_first=True
+        # )
+        # self.transformer_encoder = nn.TransformerEncoder(encoder_layer, num_layers=num_layers)
+        self.classifier = nn.Sequential(
+            nn.Linear(input_dim, 320),
+            nn.ReLU(),
+            nn.Linear(320, 1)
+        )
+        self.sigmoid = nn.Sigmoid()
+
+    def forward(self, embeddings):
+        #attn_out, _ = self.self_attention(embeddings, embeddings, embeddings)
+        #transformer_out = self.transformer_encoder(attn_out)#.squeeze(1).mean(dim=1)
+        #logits = self.classifier(transformer_out)
+
+        logits = self.classifier(embeddings)
+        probs = self.sigmoid(logits.squeeze(-1))
+        
+        return probs # Get probabilities of dimension seq_len
+    
+
+# Training function
+def train(model, train_loader, val_loader, optimizer, criterion, device):
+    """
+    Trains the model for a single epoch.
+    Args:
+        model (nn.Module): model that will be trained
+        dataloader (DataLoader): PyTorch DataLoader with training data
+        optimizer (torch.optim): optimizer
+        criterion (nn.Module): loss function
+        device (torch.device): device (GPU or CPU to train the model
+    Returns:
+        total_loss (float): model loss
+    """
+    # Training loop
+    model.train()
+    train_loss = 0
+
+    prog_bar = tqdm(total=len(train_loader), leave=True, file=sys.stdout)
+    for step, batch in enumerate(train_loader, start=1):
+        embeddings, labels, seq_len = batch
+        embeddings, labels = embeddings.to(device), labels.to(device)
+        embeddings = embeddings.squeeze(1)
+        optimizer.zero_grad()
+        outputs = model(embeddings)
+        loss = criterion(outputs, labels)
+        loss.backward()
+        optimizer.step()
+        train_loss += loss.item()
+        prog_bar.update()
+        sys.stdout.flush()
+    prog_bar.close()
+
+    # Validation loop
+    model.eval()
+    val_loss = 0.0
+
+    prog_bar = tqdm(total=len(val_loader), leave=True, file=sys.stdout)
+    for step, batch in enumerate(val_loader):
+        embeddings, labels, seq_len = batch
+        embeddings, labels = embeddings.to(device), labels.to(device)
+        with torch.no_grad():
+            outputs = model(embeddings)
+            loss = criterion(outputs, labels)
+        val_loss += loss.item()
+        prog_bar.update()
+        sys.stdout.flush()
+    prog_bar.close()
+
+    return train_loss/len(train_loader), val_loss/len(val_loader)
+
+
+
+# Evaluation function
+def evaluate(model, dataloader, device):
+    """
+    Performs inference on a trained model
+    Args:
+        model (nn.Module): the trained model
+        dataloader (DataLoader): PyTorch DataLoader with testing data
+        device (torch.device): device (GPU or CPU) to be used for inference
+    Returns:
+        preds (list): predicted per-residue disorder labels
+        true_labels (list): ground truth per-residue disorder labels
+    """
+    model.eval()
+    preds, true_labels = [], []
+    with torch.no_grad():
+        for embeddings, labels, seq_len in tqdm(dataloader):
+            embeddings, labels = embeddings.to(device), labels.to(device)
+            outputs = model(embeddings)
+            preds.append(outputs.cpu().numpy())
+            true_labels.append(labels.cpu().numpy())
+    return preds, true_labels
+
+# Metrics calculation
+def calculate_metrics(preds, labels, threshold=0.5):
+    """
+    Calculates metrics to assess model performance
+    Args:
+        preds (list): model's predictions
+        labels (list): ground truth labels
+        threshold (float): minimum threshold a prediction must be met to be considered disordered
+    Returns:
+        accuracy (float): accuracy
+        precision (float): precision
+        recall (float): recall
+        f1 (float): F1 score
+        roc_auc (float): AUROC score
+    """
+    flat_binary_preds, flat_prob_preds, flat_labels = [], [], []
+
+    for pred, label in zip(preds, labels):
+        flat_binary_preds.extend((pred > threshold).astype(int).flatten())
+        flat_prob_preds.extend(pred.flatten())
+        flat_labels.extend(label.flatten())
+
+    flat_binary_preds = np.array(flat_binary_preds)
+    flat_prob_preds = np.array(flat_prob_preds)
+    flat_labels = np.array(flat_labels)
+
+    accuracy = accuracy_score(flat_labels, flat_binary_preds)
+    precision = precision_score(flat_labels, flat_binary_preds)
+    recall = recall_score(flat_labels, flat_binary_preds)
+    f1 = f1_score(flat_labels, flat_binary_preds)
+    roc_auc = roc_auc_score(flat_labels, flat_prob_preds)
+
+    return accuracy, precision, recall, f1, roc_auc
+
+
+if __name__ == "__main__":
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    print(device)
+
+    for embedding_type in ['mlm', 'esm', 'mdlm']:
+        best_val_loss = float('inf')
+        best_model = None
+
+        # Load train and test dataset
+        train_dataset = SolubilityDataset(embedding_type,
+                                          hyperparams['train_data'],
+                                          hyperparams['esm_model_path'],
+                                          hyperparams['mlm_model_path'],
+                                          hyperparams['mdlm_model_path'],
+                                          device)
+        test_dataset = SolubilityDataset(embedding_type,
+                                         hyperparams['test_data'],
+                                         hyperparams['esm_model_path'],
+                                         hyperparams['mlm_model_path'],
+                                         hyperparams['mdlm_model_path'],
+                                         device)
+        val_dataset = SolubilityDataset(embedding_type,
+                                         hyperparams['val_data'],
+                                         hyperparams['esm_model_path'],
+                                         hyperparams['mlm_model_path'],
+                                         hyperparams['mdlm_model_path'],
+                                         device)
+
+        # Load PyTorch datasets into DataLoaders
+        train_dataloader = DataLoader(train_dataset, batch_size=hyperparams["batch_size"], shuffle=True)
+        val_dataloader = DataLoader(val_dataset, batch_size=hyperparams["batch_size"], shuffle=False)
+        test_dataloader = DataLoader(test_dataset, batch_size=hyperparams["batch_size"], shuffle=False)
+
+        # Set device to GPU
+
+        ### Grid search to explore hyperparameter space
+        # Define hyperparameters
+        param_grid = {
+            'learning_rate': [5e-4],
+            'batch_size': [1],
+            'num_heads': [4],
+            'num_layers': [2],
+            'dropout': [0.5],
+            'num_epochs': [5]
+        }
+
+        # Loop over the parameter grid
+        grid = ParameterGrid(param_grid)
+        for params in grid:
+            # Update hyperparameters
+            hyperparams.update(params)
+            
+            # Update model with the new set of hyperparms
+            input_dim=640 if embedding_type=="mdlm" else 1280
+            hidden_dim = input_dim
+            model = SolubilityPredictor(
+                input_dim=input_dim,
+                hidden_dim=hidden_dim,
+                num_layers=hyperparams["num_layers"],
+                num_heads=hyperparams["num_heads"],
+                dropout=hyperparams['dropout']
+            )
+            model = model.to(device) # Push model to GPU
+            
+            # Update optimizer
+            optimizer = optim.Adam(model.parameters(), lr=hyperparams["learning_rate"])
+            criterion = nn.BCELoss()
+            num_epochs = hyperparams['num_epochs']
+
+            # Train 
+            for epoch in range(hyperparams["num_epochs"]):
+                print(f"EPOCH {epoch+1}/{hyperparams['num_epochs']}")
+                train_loss, val_loss = train(model, train_dataloader, val_dataloader, optimizer, criterion, device)
+                print(f"TRAIN LOSS: {train_loss:.4f}")
+                print(f"VALIDATION LOSS: {val_loss:.4f}\n")
+                sys.stdout.flush()
+
+                if val_loss < best_val_loss:
+                    best_val_loss = val_loss
+                    best_model = model.state_dict()
+
+            # Evaluate model on test sequences
+            print("TEST METRICS:")
+            test_preds, test_labels = evaluate(model, test_dataloader, device)
+            test_metrics = calculate_metrics(test_preds, test_labels)
+            print(f"Accuracy: {test_metrics[0]:.4f}")
+            print(f"Precision: {test_metrics[1]:.4f}")
+            print(f"Recall: {test_metrics[2]:.4f}")
+            print(f"F1 Score: {test_metrics[3]:.4f}")
+            print(f"ROC AUC: {test_metrics[4]:.4f}")
+            print(f"\n")
+            sys.stdout.flush()
+
+            ### Save model and metrics for this hyperparameter combination
+            folder_name = f"{path}/benchmarks/Supervised/Solubility/transformer_models/{embedding_type}/lr{hyperparams['learning_rate']}_bs{hyperparams['batch_size']}_epochs{hyperparams['num_epochs']}_layers{hyperparams['num_layers']}_heads{hyperparams['num_heads']}_drpt{hyperparams['dropout']}"
+            os.makedirs(folder_name, exist_ok=True)
+
+            # Save current model for this hyperparameter combination
+            model_file_path = os.path.join(folder_name, "model.pth")
+            torch.save(model.state_dict(), model_file_path)
+
+            # Save hyperparameters and test metrics to txt file
+            output_file_path = os.path.join(folder_name, "hyperparams_and_test_results.txt")
+            with open(output_file_path, 'w') as out_file:
+                for key, value in hyperparams.items():
+                    out_file.write(f"{key}: {value}\n")
+                
+                out_file.write("\nTEST METRICS:\n")
+                out_file.write(f"Accuracy: {test_metrics[0]:.4f}\n")
+                out_file.write(f"Precision: {test_metrics[1]:.4f}\n")
+                out_file.write(f"Recall: {test_metrics[2]:.4f}\n")
+                out_file.write(f"F1 Score: {test_metrics[3]:.4f}\n")
+                out_file.write(f"ROC AUC: {test_metrics[4]:.4f}\n")
+
+        # Save the best model and its hyperparameters
+        if best_model is not None:
+            best_model_dir = f"{path}/benchmarks/Supervised/Solubility/transformer_models/{embedding_type}"
+            os.makedirs(best_model_dir, exist_ok=True)
+            best_model_path = os.path.join(best_model_dir, "best_model.pth")
+            torch.save(best_model, best_model_path)
+
+            # Save the hyperparameters for the best model
+            best_hyperparams_path = f"{path}/benchmarks/Supervised/Solubility/transformer_models/{embedding_type}/best_model_hyperparams.txt"
+            with open(best_hyperparams_path, 'w') as out_file:
+                out_file.write("Best Validation Loss: {:.4f}\n".format(best_val_loss))
+                for key, value in hyperparams.items():
+                    out_file.write(f"{key}: {value}\n")